diff --git a/README.md b/README.md index 6d1faddcf..6990f5824 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,8 @@ uv add langwatch-scenario pytest pnpm install @langwatch/scenario vitest ``` +The TypeScript package targets [AI SDK](https://ai-sdk.dev) 7. If your project is still on AI SDK 6, stay on `@langwatch/scenario` 1.x until you upgrade. + Now create your first scenario, copy the full working example below.
@@ -198,13 +200,8 @@ describe("Vegetarian Recipe Agent", () => { call: async (input) => { const response = await generateText({ model: openai("gpt-4.1-mini"), - messages: [ - { - role: "system", - content: `You are a vegetarian recipe agent.\nGiven the user request, ask AT MOST ONE follow-up question, then provide a complete recipe. Keep your responses concise and focused.`, - }, - ...input.messages, - ], + instructions: `You are a vegetarian recipe agent.\nGiven the user request, ask AT MOST ONE follow-up question, then provide a complete recipe. Keep your responses concise and focused.`, + messages: input.messages, }); return response.text; }, diff --git a/docs/docs/pages/advanced/remote-traces.mdx b/docs/docs/pages/advanced/remote-traces.mdx index 37a7c8c3a..a69cb4411 100644 --- a/docs/docs/pages/advanced/remote-traces.mdx +++ b/docs/docs/pages/advanced/remote-traces.mdx @@ -238,13 +238,23 @@ async def test_remote_trace_propagation(): ```typescript [typescript] import { createServer, Server } from "http"; import { openai } from "@ai-sdk/openai"; +import { OpenTelemetry } from "@ai-sdk/otel"; import scenario, { AgentRole, type AgentAdapter } from "@langwatch/scenario"; import { context, propagation } from "@opentelemetry/api"; import { W3CTraceContextPropagator } from "@opentelemetry/core"; -import { generateText, ModelMessage, stepCountIs, tool } from "ai"; +import { + generateText, + ModelMessage, + isStepCount, + registerTelemetry, + tool, +} from "ai"; import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { z } from "zod/v4"; +// AI SDK 7 emits no spans until a telemetry integration is registered. +registerTelemetry(new OpenTelemetry()); + // Standard OTel HTTP instrumentation registers a propagator on its own; // this hand-rolled demo server registers the W3C propagator explicitly. propagation.setGlobalPropagator(new W3CTraceContextPropagator()); @@ -277,17 +287,12 @@ beforeAll(async () => { const result = await context.with(extracted, () => generateText({ model: openai("gpt-5-mini"), - messages: [ - { - role: "system", - content: - "You are an order support assistant. Use the lookup_order_status tool to check orders before answering.", - }, - ...messages, - ], + instructions: + "You are an order support assistant. Use the lookup_order_status tool to check orders before answering.", + messages, tools: { lookup_order_status: lookupOrderStatus }, - stopWhen: stepCountIs(3), - experimental_telemetry: { isEnabled: true }, + stopWhen: isStepCount(3), + telemetry: { isEnabled: true }, }) ); diff --git a/docs/docs/pages/agent-integration.mdx b/docs/docs/pages/agent-integration.mdx index 462afc675..531ad285e 100644 --- a/docs/docs/pages/agent-integration.mdx +++ b/docs/docs/pages/agent-integration.mdx @@ -379,7 +379,7 @@ import { z } from "zod"; const getCurrentWeather = tool({ description: "Get the current weather in a given city.", - parameters: z.object({ + inputSchema: z.object({ city: z.string().describe("The city to get the weather for."), }), execute: async ({ city }) => `The weather in ${city} is sunny!`, diff --git a/docs/docs/pages/agent-integration/vercel-ai.mdx b/docs/docs/pages/agent-integration/vercel-ai.mdx index df5f38dad..1b8ea1631 100644 --- a/docs/docs/pages/agent-integration/vercel-ai.mdx +++ b/docs/docs/pages/agent-integration/vercel-ai.mdx @@ -20,11 +20,8 @@ import { generateText } from 'ai'; async function myAgent(prompt: string) { const result = await generateText({ model: openai('gpt-4-turbo'), + instructions: 'You are a helpful AI assistant.', messages: [ - { - role: 'system', - content: 'You are a helpful AI assistant.', - }, { role: 'user', content: prompt, @@ -46,7 +43,7 @@ const createMyAgentAdapter = (): AgentAdapter => { name: "VercelAIAgent", call: async (input) => { const result = await myAgent(input.messages.at(-1)?.content); - return result.response.messages; + return result.responseMessages; }, }; }; diff --git a/docs/docs/pages/basics/scripted-simulations.mdx b/docs/docs/pages/basics/scripted-simulations.mdx index c378d8214..a4f218997 100644 --- a/docs/docs/pages/basics/scripted-simulations.mdx +++ b/docs/docs/pages/basics/scripted-simulations.mdx @@ -257,7 +257,8 @@ const script = [ ], }), - // System message + // System message (with AI SDK 7, an agent that forwards these messages to + // generateText needs `allowSystemInMessages: true`) message({ role: "system", content: "The user is now in a hurry and needs quick responses", diff --git a/docs/docs/pages/examples/multimodal/multimodal-images.mdx b/docs/docs/pages/examples/multimodal/multimodal-images.mdx index 3e8553482..a6f629bfc 100644 --- a/docs/docs/pages/examples/multimodal/multimodal-images.mdx +++ b/docs/docs/pages/examples/multimodal/multimodal-images.mdx @@ -36,13 +36,8 @@ const imageAgent: AgentAdapter = { async call(input) { const response = await generateText({ model: openai("gpt-4o"), - messages: [ - { - role: "system", - content: `You are a helpful assistant that can describe images.`, - }, - ...input.messages, - ], + instructions: `You are a helpful assistant that can describe images.`, + messages: input.messages, }); return response.text; }, diff --git a/docs/docs/pages/introduction/getting-started.mdx b/docs/docs/pages/introduction/getting-started.mdx index 279fb0b95..7e79d85b8 100644 --- a/docs/docs/pages/introduction/getting-started.mdx +++ b/docs/docs/pages/introduction/getting-started.mdx @@ -26,6 +26,40 @@ pnpm install @langwatch/scenario vitest ::: + +The TypeScript SDK targets [AI SDK](https://ai-sdk.dev) 7 and Node.js 22.12 or later. If your project is still on AI SDK 6, stay on `@langwatch/scenario` 1.x until you upgrade. + + +### AI SDK telemetry (TypeScript) + +AI SDK 7 emits no spans until a telemetry integration is registered, so the judge and LangWatch cannot see your agent's LLM calls by default. Register the OpenTelemetry integration once at start-up, for example in a Vitest setup file: + +```bash +pnpm install @ai-sdk/otel +``` + +```typescript +// vitest.setup.ts +import { OpenTelemetry } from "@ai-sdk/otel"; +import { registerTelemetry } from "ai"; + +registerTelemetry(new OpenTelemetry()); +``` + +To attach per-call metadata to the spans (a user id, a tenant, a feature flag), pass it as `runtimeContext` and opt each key in with `telemetry.includeRuntimeContext`. Turn on `runtimeContext` on the integration too; the values are emitted as `ai.settings.context.`: + +```typescript +registerTelemetry(new OpenTelemetry({ runtimeContext: true })); + +await generateText({ + model: openai("gpt-5-mini"), + instructions: "You are a vegetarian recipe agent.", + messages: input.messages, + runtimeContext: { userId: "user-123" }, + telemetry: { functionId: "recipe-agent", includeRuntimeContext: { userId: true } }, +}); +``` + ## Quick Start Create your first Scenario test. This example demonstrates testing a vegetarian recipe agent: @@ -138,10 +172,8 @@ const vegetarianRecipeAgent: AgentAdapter = { call: async (input) => { const response = await generateText({ model: openai("gpt-4.1-mini"), - messages: [ - { role: "system", content: "You are a vegetarian recipe agent." }, - ...input.messages, - ], + instructions: "You are a vegetarian recipe agent.", + messages: input.messages, }); return response.text; }, diff --git a/docs/docs/pages/testing-guides/mocks.mdx b/docs/docs/pages/testing-guides/mocks.mdx index 3a5e94991..f104d4d69 100644 --- a/docs/docs/pages/testing-guides/mocks.mdx +++ b/docs/docs/pages/testing-guides/mocks.mdx @@ -161,7 +161,7 @@ const mockWeatherTool = vi.fn().mockResolvedValue("Sunny, 75°F"); const weatherTool = tool({ description: "Get weather data", - parameters: z.object({ city: z.string() }), + inputSchema: z.object({ city: z.string() }), execute: mockWeatherTool, }); diff --git a/docs/docs/pages/testing-guides/tool-calling.mdx b/docs/docs/pages/testing-guides/tool-calling.mdx index fd5c4ded9..4bf2138af 100644 --- a/docs/docs/pages/testing-guides/tool-calling.mdx +++ b/docs/docs/pages/testing-guides/tool-calling.mdx @@ -309,16 +309,11 @@ const agent = (): AgentAdapter => ({ call: async (input) => { const response = await generateText({ model: openai("gpt-4.1-nano"), - messages: [ - { - role: "system", - content: ` + instructions: ` You are a helpful assistant that may help the user with weather information. Do not guess the city if they don't provide it. `, - }, - ...input.messages, - ], + messages: input.messages, tools: { get_current_weather: getCurrentWeather }, toolChoice: "auto", }); diff --git a/javascript/README.md b/javascript/README.md index 8714e59da..afb6d9b42 100644 --- a/javascript/README.md +++ b/javascript/README.md @@ -27,6 +27,17 @@ npm install @langwatch/scenario yarn add @langwatch/scenario ``` +Scenario targets [AI SDK](https://ai-sdk.dev) 7 (`ai@^7`) and Node.js 22.12 or later. If your project is still on AI SDK 6, stay on `@langwatch/scenario` 1.x until you upgrade. + +AI SDK 7 emits no spans until a telemetry integration is registered. To let the judge and LangWatch see your agent's LLM calls, register one once at start-up (for example in a Vitest setup file): + +```typescript +import { OpenTelemetry } from "@ai-sdk/otel"; +import { registerTelemetry } from "ai"; + +registerTelemetry(new OpenTelemetry()); +``` + ## Quick Start Create your first scenario test in under a minute. @@ -86,7 +97,7 @@ describe("Weather Agent", () => { // 1. Define the tools your agent can use const getCurrentWeather = tool({ description: "Get the current weather in a given city.", - parameters: z.object({ + inputSchema: z.object({ city: z.string().describe("The city to get the weather for."), }), execute: async ({ city }) => @@ -100,7 +111,7 @@ describe("Weather Agent", () => { call: async (input) => { const response = await generateText({ model: openai("gpt-4.1-mini"), - system: `You are a helpful assistant that may help the user with weather information.`, + instructions: `You are a helpful assistant that may help the user with weather information.`, messages: input.messages, tools: { get_current_weather: getCurrentWeather }, }); @@ -113,6 +124,7 @@ describe("Weather Agent", () => { { toolCallId: toolCall.toolCallId, messages: input.messages, + context: {}, } ); return [ diff --git a/javascript/examples/vitest/package.json b/javascript/examples/vitest/package.json index 2b2737a67..d90555d02 100644 --- a/javascript/examples/vitest/package.json +++ b/javascript/examples/vitest/package.json @@ -21,11 +21,12 @@ "vitest-mock-extended": "5.1.1" }, "dependencies": { - "@ai-sdk/anthropic": "3.0.41", - "@ai-sdk/xai": "3.0.50", + "@ai-sdk/anthropic": "4.0.79", + "@ai-sdk/otel": "^1.0.130", + "@ai-sdk/xai": "5.0.21", "@langwatch/scenario": "workspace:*", "@openai/agents": "^0.16.0", - "ai": "^6.0.0", + "ai": "^7.0.130", "express": "^5.1.0", "openai": "^6.9.1", "vite": "^8.2.1", diff --git a/javascript/examples/vitest/tests/00-demo-light.test.ts b/javascript/examples/vitest/tests/00-demo-light.test.ts index df0857c6c..07811b718 100644 --- a/javascript/examples/vitest/tests/00-demo-light.test.ts +++ b/javascript/examples/vitest/tests/00-demo-light.test.ts @@ -25,15 +25,10 @@ describe("Demo: Lightweight Scenarios", () => { call: async (input) => { const response = await generateText({ model: openai("gpt-5-mini"), - messages: [ - { - role: "system", - content: - "You can analyze images and respond briefly and helpfully.", - }, - ...input.messages, - ], - experimental_telemetry: { isEnabled: true }, + instructions: + "You can analyze images and respond briefly and helpfully.", + messages: input.messages, + telemetry: { isEnabled: true }, }); return response.text; }, diff --git a/javascript/examples/vitest/tests/api-service-mocking.test.ts b/javascript/examples/vitest/tests/api-service-mocking.test.ts index 76b9e29bd..aea42f2c0 100644 --- a/javascript/examples/vitest/tests/api-service-mocking.test.ts +++ b/javascript/examples/vitest/tests/api-service-mocking.test.ts @@ -48,7 +48,7 @@ const userDataAgent: AgentAdapter = { messages: input.messages, tools: { fetch_user_data: fetchUserDataTool }, toolChoice: "auto", - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); return response.text; }, diff --git a/javascript/examples/vitest/tests/database-tool-mocking.test.ts b/javascript/examples/vitest/tests/database-tool-mocking.test.ts index 29e8b4a75..bbcefc800 100644 --- a/javascript/examples/vitest/tests/database-tool-mocking.test.ts +++ b/javascript/examples/vitest/tests/database-tool-mocking.test.ts @@ -52,7 +52,7 @@ const databaseAgent: AgentAdapter = { find_user: findUserTool, }, toolChoice: "auto", - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); return response.text; }, diff --git a/javascript/examples/vitest/tests/false-assumptions.test.ts b/javascript/examples/vitest/tests/false-assumptions.test.ts index 1921482ee..9e14065c9 100644 --- a/javascript/examples/vitest/tests/false-assumptions.test.ts +++ b/javascript/examples/vitest/tests/false-assumptions.test.ts @@ -16,11 +16,9 @@ describe("False Assumptions", () => { call: async (input) => { const response = await generateText({ model: openai("gpt-5-mini"), - messages: [ - { role: "system", content: "You are a helpful assistant" }, - ...input.messages, - ], - experimental_telemetry: { isEnabled: true }, + instructions: "You are a helpful assistant", + messages: input.messages, + telemetry: { isEnabled: true }, }); return response.text; diff --git a/javascript/examples/vitest/tests/helpers/register-ai-telemetry.ts b/javascript/examples/vitest/tests/helpers/register-ai-telemetry.ts new file mode 100644 index 000000000..f29de13b5 --- /dev/null +++ b/javascript/examples/vitest/tests/helpers/register-ai-telemetry.ts @@ -0,0 +1,8 @@ +/** + * AI SDK 7 emits no spans until a telemetry integration is registered. + * Loaded once per worker as a vitest setup file. + */ +import { OpenTelemetry } from "@ai-sdk/otel"; +import { registerTelemetry } from "ai"; + +registerTelemetry(new OpenTelemetry()); diff --git a/javascript/examples/vitest/tests/judge-tool-calls-without-telemetry.test.ts b/javascript/examples/vitest/tests/judge-tool-calls-without-telemetry.test.ts index e818c0cd5..8da890d09 100644 --- a/javascript/examples/vitest/tests/judge-tool-calls-without-telemetry.test.ts +++ b/javascript/examples/vitest/tests/judge-tool-calls-without-telemetry.test.ts @@ -42,7 +42,7 @@ const weatherAgent: AgentAdapter = { messages: input.messages, tools: { get_weather: getWeatherTool }, toolChoice: "auto", - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); if (response.toolCalls && response.toolCalls.length > 0) { diff --git a/javascript/examples/vitest/tests/llm-provider-mocking.test.ts b/javascript/examples/vitest/tests/llm-provider-mocking.test.ts index ebbf30132..7d2aaecc3 100644 --- a/javascript/examples/vitest/tests/llm-provider-mocking.test.ts +++ b/javascript/examples/vitest/tests/llm-provider-mocking.test.ts @@ -1,6 +1,6 @@ import { openai } from "@ai-sdk/openai"; import scenario, { type AgentAdapter, AgentRole } from "@langwatch/scenario"; -import { generateText, ToolSet, GenerateTextResult } from "ai"; +import { generateText } from "ai"; import { describe, it, expect, vi } from "vitest"; /** @@ -39,7 +39,7 @@ const chatAgent: AgentAdapter = { const response = await generateText({ model: openai("gpt-5-mini"), messages: input.messages, - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); return response.text; }, @@ -51,7 +51,7 @@ describe("LLM Provider Mocking", () => { // No actual LLM call will be made mockGenerateText.mockResolvedValue({ text: "I can help you with that request.", - } as GenerateTextResult); + } as Awaited>); const result = await scenario.run({ name: "llm mock test", diff --git a/javascript/examples/vitest/tests/mocked-weather-agent-tool.test.ts b/javascript/examples/vitest/tests/mocked-weather-agent-tool.test.ts index ba6a7312f..1f4594778 100644 --- a/javascript/examples/vitest/tests/mocked-weather-agent-tool.test.ts +++ b/javascript/examples/vitest/tests/mocked-weather-agent-tool.test.ts @@ -26,19 +26,14 @@ describe("Mocked Weather Agent Tool", () => { call: async (input) => { const response = await generateText({ model: openai("gpt-5-mini"), - messages: [ - { - role: "system", - content: ` + instructions: ` You are a helpful assistant that may help the user with weather information. Do not guess the city if they don't provide it. `, - }, - ...input.messages, - ], + messages: input.messages, tools: { get_current_weather: getCurrentWeather }, toolChoice: "auto", - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); return response.text; diff --git a/javascript/examples/vitest/tests/multilingual-agent.test.ts b/javascript/examples/vitest/tests/multilingual-agent.test.ts index 0cbc0e361..89b58f879 100644 --- a/javascript/examples/vitest/tests/multilingual-agent.test.ts +++ b/javascript/examples/vitest/tests/multilingual-agent.test.ts @@ -16,7 +16,7 @@ const createMultilingualAgent = (): AgentAdapter => ({ const response = await generateText({ model: openai("gpt-5-mini"), messages: input.messages, - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); return response.text; diff --git a/javascript/examples/vitest/tests/multimodal-images.test.ts b/javascript/examples/vitest/tests/multimodal-images.test.ts index 27f7f1f29..693e2e7dc 100644 --- a/javascript/examples/vitest/tests/multimodal-images.test.ts +++ b/javascript/examples/vitest/tests/multimodal-images.test.ts @@ -35,18 +35,13 @@ describe("Multimodal Images Tests", () => { call: async (input) => { const response = await generateText({ model: openai("gpt-5-mini"), - messages: [ - { - role: "system", - content: ` + instructions: ` You are a helpful assistant that can process both text and image input. When analyzing images, be descriptive and helpful. Respond naturally to user queries about images. `, - }, - ...input.messages, - ], - experimental_telemetry: { isEnabled: true }, + messages: input.messages, + telemetry: { isEnabled: true }, }); return response.text; }, diff --git a/javascript/examples/vitest/tests/running-in-parallel.test.ts b/javascript/examples/vitest/tests/running-in-parallel.test.ts index b5b67fb27..9c392496e 100644 --- a/javascript/examples/vitest/tests/running-in-parallel.test.ts +++ b/javascript/examples/vitest/tests/running-in-parallel.test.ts @@ -9,18 +9,13 @@ describe("Vegetarian Recipe Agent (Parallel)", () => { call: async (input) => { const response = await generateText({ model: openai("gpt-5-mini"), - messages: [ - { - role: "system", - content: `You are a vegetarian recipe agent. When a user asks for a recipe, + instructions: `You are a vegetarian recipe agent. When a user asks for a recipe, - You may ask ONLY ONE clarifying question if absolutely necessary (like dietary restrictions or cooking time preference), but then you MUST provide a complete vegetarian recipe. - Do not ask multiple follow-up questions. `, - }, - ...input.messages, - ], - experimental_telemetry: { isEnabled: true }, + messages: input.messages, + telemetry: { isEnabled: true }, }); return response.text; }, diff --git a/javascript/examples/vitest/tests/simple-tool-mocking.test.ts b/javascript/examples/vitest/tests/simple-tool-mocking.test.ts index 2f4907a2f..c9d54a3ea 100644 --- a/javascript/examples/vitest/tests/simple-tool-mocking.test.ts +++ b/javascript/examples/vitest/tests/simple-tool-mocking.test.ts @@ -39,7 +39,7 @@ const userDataAgent: AgentAdapter = { messages: input.messages, tools: { fetch_user_data: fetchUserDataTool }, toolChoice: "auto", - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); return response.text; }, diff --git a/javascript/examples/vitest/tests/span-based-evaluation.test.ts b/javascript/examples/vitest/tests/span-based-evaluation.test.ts index a59dcade9..7bd4c1a08 100644 --- a/javascript/examples/vitest/tests/span-based-evaluation.test.ts +++ b/javascript/examples/vitest/tests/span-based-evaluation.test.ts @@ -1,7 +1,7 @@ import { openai } from "@ai-sdk/openai"; import scenario, { type AgentAdapter, AgentRole } from "@langwatch/scenario"; import { trace, context } from "@opentelemetry/api"; -import { generateText, stepCountIs, tool } from "ai"; +import { generateText, isStepCount, tool } from "ai"; import { describe, it, expect } from "vitest"; import { z } from "zod/v4"; @@ -68,17 +68,12 @@ const observableAgent: AgentAdapter = { // LLM call with tool usage const response = await generateText({ model: openai("gpt-5-mini"), - messages: [ - { - role: "system", - content: `You are an order processing assistant. + instructions: `You are an order processing assistant. When asked about products, use the check_inventory tool.`, - }, - ...input.messages, - ], + messages: input.messages, tools: { check_inventory: checkInventoryTool }, - experimental_telemetry: { isEnabled: true }, - stopWhen: stepCountIs(2), + telemetry: { isEnabled: true }, + stopWhen: isStepCount(2), }); return response.text; diff --git a/javascript/examples/vitest/tests/testing-remote-agents-json.test.ts b/javascript/examples/vitest/tests/testing-remote-agents-json.test.ts index 010710ff9..24e923fc7 100644 --- a/javascript/examples/vitest/tests/testing-remote-agents-json.test.ts +++ b/javascript/examples/vitest/tests/testing-remote-agents-json.test.ts @@ -31,21 +31,18 @@ beforeAll(async () => { // Use real LLM to generate response const result = await generateText({ model: openai("gpt-5-mini"), + /** + * Simple system prompt to make the agent pretend like it has access to a weather API and make up the weather so the tests pass. + */ + instructions: + "You are a helpful weather assistant. Provide brief, friendly responses about weather. Pretend like you have access to a weather API and make up the weather.", messages: [ - { - role: "system", - /** - * Simple system prompt to make the agent pretend like it has access to a weather API and make up the weather so the tests pass. - */ - content: - "You are a helpful weather assistant. Provide brief, friendly responses about weather. Pretend like you have access to a weather API and make up the weather.", - }, { role: "user", content: message, }, ], - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); res.writeHead(200, { "Content-Type": "application/json" }); diff --git a/javascript/examples/vitest/tests/testing-remote-agents-sse.test.ts b/javascript/examples/vitest/tests/testing-remote-agents-sse.test.ts index 505c406db..be84eb6a6 100644 --- a/javascript/examples/vitest/tests/testing-remote-agents-sse.test.ts +++ b/javascript/examples/vitest/tests/testing-remote-agents-sse.test.ts @@ -44,18 +44,15 @@ beforeAll(async () => { // Stream response using real LLM const result = streamText({ model: openai("gpt-5-mini"), + instructions: + "You are a helpful weather assistant. Provide brief, friendly responses. Pretend you have access to weather data. Pretend like you have access to a weather API and make up the weather.", messages: [ - { - role: "system", - content: - "You are a helpful weather assistant. Provide brief, friendly responses. Pretend you have access to weather data. Pretend like you have access to a weather API and make up the weather.", - }, { role: "user", content, }, ], - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); // Stream chunks in SSE format diff --git a/javascript/examples/vitest/tests/testing-remote-agents-stateful.test.ts b/javascript/examples/vitest/tests/testing-remote-agents-stateful.test.ts index ee440c05d..eda2e0dad 100644 --- a/javascript/examples/vitest/tests/testing-remote-agents-stateful.test.ts +++ b/javascript/examples/vitest/tests/testing-remote-agents-stateful.test.ts @@ -42,15 +42,10 @@ beforeAll(async () => { // Generate response with full history const result = await generateText({ model: openai("gpt-5-mini"), - messages: [ - { - role: "system", - content: - "You are a helpful travel assistant. Help users plan their trips and provide information about destinations.", - }, - ...history, - ], - experimental_telemetry: { isEnabled: true }, + instructions: + "You are a helpful travel assistant. Help users plan their trips and provide information about destinations.", + messages: history, + telemetry: { isEnabled: true }, }); // Add assistant response to history diff --git a/javascript/examples/vitest/tests/testing-remote-agents-streaming.test.ts b/javascript/examples/vitest/tests/testing-remote-agents-streaming.test.ts index 0988e0994..caa2c33d1 100644 --- a/javascript/examples/vitest/tests/testing-remote-agents-streaming.test.ts +++ b/javascript/examples/vitest/tests/testing-remote-agents-streaming.test.ts @@ -42,18 +42,15 @@ beforeAll(async () => { const result = streamText({ model: openai("gpt-5-mini"), + instructions: + "You are a helpful weather assistant. Provide brief, friendly responses, immediately. Pretend like you have access to a weather API and make up the weather.", messages: [ - { - role: "system", - content: - "You are a helpful weather assistant. Provide brief, friendly responses, immediately. Pretend like you have access to a weather API and make up the weather.", - }, { role: "user", content, }, ], - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); // Stream chunks to client diff --git a/javascript/examples/vitest/tests/testing-remote-agents-traces.test.ts b/javascript/examples/vitest/tests/testing-remote-agents-traces.test.ts index 12b2d0a4e..bdc4eb2b8 100644 --- a/javascript/examples/vitest/tests/testing-remote-agents-traces.test.ts +++ b/javascript/examples/vitest/tests/testing-remote-agents-traces.test.ts @@ -3,7 +3,7 @@ import { openai } from "@ai-sdk/openai"; import scenario, { AgentRole, type AgentAdapter } from "@langwatch/scenario"; import { context, propagation } from "@opentelemetry/api"; import { W3CTraceContextPropagator } from "@opentelemetry/core"; -import { generateText, ModelMessage, stepCountIs, tool } from "ai"; +import { generateText, ModelMessage, isStepCount, tool } from "ai"; import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { z } from "zod/v4"; @@ -60,17 +60,12 @@ beforeAll(async () => { const result = await context.with(extracted, () => generateText({ model: openai("gpt-5-mini"), - messages: [ - { - role: "system", - content: - "You are an order support assistant. Use the lookup_order_status tool to check orders before answering.", - }, - ...messages, - ], + instructions: + "You are an order support assistant. Use the lookup_order_status tool to check orders before answering.", + messages: messages, tools: { lookup_order_status: lookupOrderStatus }, - stopWhen: stepCountIs(3), - experimental_telemetry: { isEnabled: true }, + stopWhen: isStepCount(3), + telemetry: { isEnabled: true }, }) ); diff --git a/javascript/examples/vitest/tests/tool-call-role-reversal.test.ts b/javascript/examples/vitest/tests/tool-call-role-reversal.test.ts index 1d7fe42e6..9a82fdbd7 100644 --- a/javascript/examples/vitest/tests/tool-call-role-reversal.test.ts +++ b/javascript/examples/vitest/tests/tool-call-role-reversal.test.ts @@ -26,23 +26,19 @@ const toolCallingAgent: AgentAdapter = { call: async (input) => { const response = await generateText({ model: openai("gpt-5-mini"), - messages: [ - { - role: "system", - content: "You are a helpful shopping assistant. Always use the lookup tool to check product details before responding.", - }, - ...input.messages, - ], + instructions: + "You are a helpful shopping assistant. Always use the lookup tool to check product details before responding.", + messages: input.messages, tools: { lookup_product: lookupTool }, toolChoice: "auto", - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); if (response.toolCalls && response.toolCalls.length > 0) { const toolCall = response.toolCalls[0]; const toolResult = await lookupTool.execute!( toolCall.input as { product: string }, - { toolCallId: toolCall.toolCallId, messages: input.messages } + { toolCallId: toolCall.toolCallId, messages: input.messages, context: {} } ); return [ { diff --git a/javascript/examples/vitest/tests/tool-failure-simulation.test.ts b/javascript/examples/vitest/tests/tool-failure-simulation.test.ts index 8f7c2587e..af1848848 100644 --- a/javascript/examples/vitest/tests/tool-failure-simulation.test.ts +++ b/javascript/examples/vitest/tests/tool-failure-simulation.test.ts @@ -45,7 +45,7 @@ const resilientAgent: AgentAdapter = { messages: input.messages, tools: { call_external_service: callExternalServiceTool }, toolChoice: "auto", - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); // Check if the LLM decided to call any tools diff --git a/javascript/examples/vitest/tests/vegetarian-recipe-agent.test.ts b/javascript/examples/vitest/tests/vegetarian-recipe-agent.test.ts index b8e4756d4..f1ea44970 100644 --- a/javascript/examples/vitest/tests/vegetarian-recipe-agent.test.ts +++ b/javascript/examples/vitest/tests/vegetarian-recipe-agent.test.ts @@ -9,19 +9,14 @@ describe("Vegetarian Recipe Agent", () => { call: async (input) => { const response = await generateText({ model: openai("gpt-5-mini"), - messages: [ - { - role: "system", - content: ` + instructions: ` You are a vegetarian recipe agent. Given the user request, ask AT MOST ONE follow-up question, then provide a complete recipe. Once you have given a recipe, answer without asking further questions. Keep your responses concise and focused. `, - }, - ...input.messages, - ], - experimental_telemetry: { isEnabled: true }, + messages: input.messages, + telemetry: { isEnabled: true }, }); return response.text; }, diff --git a/javascript/examples/vitest/tests/weather-agent.test.ts b/javascript/examples/vitest/tests/weather-agent.test.ts index dab160019..f3334d2e4 100644 --- a/javascript/examples/vitest/tests/weather-agent.test.ts +++ b/javascript/examples/vitest/tests/weather-agent.test.ts @@ -24,19 +24,14 @@ const weatherAgent: AgentAdapter = { call: async (input) => { const response = await generateText({ model: openai("gpt-5-mini"), - messages: [ - { - role: "system", - content: ` + instructions: ` You are a helpful assistant that may help the user with weather information. Do not guess the city if they don't provide it. `, - }, - ...input.messages, - ], + messages: input.messages, tools: { get_current_weather: getCurrentWeather }, toolChoice: "auto", - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); if (response.toolCalls && response.toolCalls.length > 0) { @@ -47,6 +42,7 @@ const weatherAgent: AgentAdapter = { { toolCallId: toolCall.toolCallId, messages: input.messages, + context: {}, } ); return [ diff --git a/javascript/examples/vitest/vitest.config.ts b/javascript/examples/vitest/vitest.config.ts index ff8f9830d..6034fa312 100644 --- a/javascript/examples/vitest/vitest.config.ts +++ b/javascript/examples/vitest/vitest.config.ts @@ -39,8 +39,10 @@ export default withScenario( testTimeout: 180_000, // 3 minutes // Load environment variables from .env file - // Required for API keys and configuration needed by AI agents - setupFiles: ["dotenv/config"], + // Required for API keys and configuration needed by AI agents. + // Then register the AI SDK 7 OpenTelemetry integration so agent + // calls emit spans the judge and LangWatch can see. + setupFiles: ["dotenv/config", "./tests/helpers/register-ai-telemetry.ts"], // (Optional) Use Scenario's custom reporter for better test output // Provides detailed information about agent interactions, diff --git a/javascript/package.json b/javascript/package.json index 19a252bdf..9b26a2b1f 100644 --- a/javascript/package.json +++ b/javascript/package.json @@ -50,24 +50,24 @@ "author": "LangWatch", "license": "Apache-2.0", "engines": { - "node": ">=20", + "node": ">=22.12", "pnpm": ">=8" }, "packageManager": "pnpm@10.22.0", "dependencies": { "@ag-ui/core": "^0.0.58", - "@ai-sdk/openai": "^3.0.26", + "@ai-sdk/openai": "^4.0.86", "@openai/agents": "^0.16.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/context-async-hooks": "2.10.0", "@opentelemetry/sdk-node": "0.221.0", "@opentelemetry/sdk-trace-base": "2.10.0", "@opentelemetry/sdk-trace-node": "2.10.0", - "ai": "^6.0.0", + "ai": "^7.0.130", "chalk": "^5.6.2", "ffmpeg-static": "5.3.0", "fft.js": "4.0.4", - "langwatch": ">=0.16.1 <2.0.0", + "langwatch": "^1.20.0", "open": "11.0.1", "openai": "^6.9.1", "rxjs": "^7.8.2", @@ -76,8 +76,8 @@ "zod": "^3.25.76" }, "devDependencies": { - "@ai-sdk/amazon-bedrock": "^4.0.183", - "@ai-sdk/google": "^3.0.127", + "@ai-sdk/amazon-bedrock": "^5.0.115", + "@ai-sdk/google": "^4.0.95", "@amiceli/vitest-cucumber": "^7.0.0", "@elevenlabs/elevenlabs-js": "^2.51.0", "@eslint/js": "^9.39.1", @@ -132,7 +132,7 @@ "peerDependencies": { "@elevenlabs/elevenlabs-js": ">=2.51.0", "@google/genai": ">=2.0.0", - "ai": ">=6.0.0", + "ai": ">=7.0.0", "vitest": ">=3.2.4" }, "peerDependenciesMeta": { diff --git a/javascript/src/agents/__tests__/realtime-echo-safe.test.ts b/javascript/src/agents/__tests__/realtime-echo-safe.test.ts index 06c5bb135..b406d9d26 100644 --- a/javascript/src/agents/__tests__/realtime-echo-safe.test.ts +++ b/javascript/src/agents/__tests__/realtime-echo-safe.test.ts @@ -177,11 +177,11 @@ describe("realtime echo-safety (AC-JS1')", () => { // red-capable at the JS layer: the realtime turn goes straight through the // production reversal, and the parrot sees the agent's transcript verbatim. const naiveReversed = messageRoleReversal([ - { role: "system", content: "you are pretending to be a user" }, { role: "assistant", content: "Hello, how can I help you today" }, agentTurn, ]); const naiveResult = await parrotInvokeLLM({ + instructions: "you are pretending to be a user", messages: naiveReversed, } as unknown as InvokeLLMParams); const naiveAnswer = naiveResult.text ?? ""; diff --git a/javascript/src/agents/__tests__/user-simulator-instructions.test.ts b/javascript/src/agents/__tests__/user-simulator-instructions.test.ts new file mode 100644 index 000000000..65eb18d2f --- /dev/null +++ b/javascript/src/agents/__tests__/user-simulator-instructions.test.ts @@ -0,0 +1,90 @@ +/** + * The user simulator sends its system prompt as AI SDK 7 `instructions`. + * + * Covers the user simulator @unit scenarios of + * `specs/ai-sdk-7-system-prompts.feature`. AI SDK 7 refuses system messages + * inside `messages` unless the call allows them. + */ +import type { ModelMessage } from "ai"; +import { describe, it, expect, vi } from "vitest"; + +import type { AgentInput } from "../../domain"; +import type { InvokeLLMParams } from "../types"; +import { userSimulatorAgent } from "../user-simulator-agent"; + +vi.mock("../../config", () => ({ + getProjectConfig: vi.fn().mockResolvedValue({ + defaultModel: { model: "openai/gpt-5-mini", temperature: 0 }, + }), +})); + +function makeInput(messages: ModelMessage[]): AgentInput { + return { + threadId: "t-instructions", + messages, + newMessages: [], + requestedRole: "User" as AgentInput["requestedRole"], + scenarioConfig: { + name: "test", + description: "A test scenario description", + } as unknown as AgentInput["scenarioConfig"], + scenarioState: {} as AgentInput["scenarioState"], + }; +} + +/** Stub the simulator's LLM and capture the parameters of its last call. */ +function captureLLM(simulator: ReturnType) { + const seen: { params?: InvokeLLMParams } = {}; + ( + simulator as unknown as { + invokeLLM: ( + params: InvokeLLMParams + ) => Promise<{ text: string; toolCalls: []; steps: [] }>; + } + ).invokeLLM = async (params) => { + seen.params = params; + return { text: "Where is my order?", toolCalls: [], steps: [] }; + }; + return seen; +} + +describe("given a user simulator agent whose model is stubbed", () => { + describe("when the simulator produces the user's turn", () => { + /** @scenario "The user simulator sends its system prompt as instructions" */ + it("carries the system prompt as instructions, not as a message", async () => { + const simulator = userSimulatorAgent({ systemPrompt: "Be a customer" }); + const seen = captureLLM(simulator); + + await simulator.call( + makeInput([{ role: "assistant", content: "Hi, how can I help?" }]) + ); + + expect(seen.params?.instructions).toBe("Be a customer"); + expect(seen.params?.messages?.some((m) => m.role === "system")).toBe( + false + ); + }); + }); + + describe("when the transcript holds a system turn added by the script", () => { + /** @scenario "A system turn added by the script still reaches the user simulator" */ + it("allows system messages and keeps the scripted turn", async () => { + const simulator = userSimulatorAgent({ systemPrompt: "Be a customer" }); + const seen = captureLLM(simulator); + const scripted: ModelMessage = { + role: "system", + content: "The user is now in a hurry", + }; + + await simulator.call( + makeInput([ + { role: "assistant", content: "Hi, how can I help?" }, + scripted, + ]) + ); + + expect(seen.params?.allowSystemInMessages).toBe(true); + expect(seen.params?.messages).toContainEqual(scripted); + }); + }); +}); diff --git a/javascript/src/agents/__tests__/user-simulator-voice.test.ts b/javascript/src/agents/__tests__/user-simulator-voice.test.ts index c1cdb6bb5..a07b09132 100644 --- a/javascript/src/agents/__tests__/user-simulator-voice.test.ts +++ b/javascript/src/agents/__tests__/user-simulator-voice.test.ts @@ -271,11 +271,10 @@ describeFeature( // Capture system prompt via LLM spy — check persona block present. systemPromptCapture = ""; (sim as unknown as { - invokeLLM: (p: { messages: Array<{ role: string; content: string }> }) => Promise<{ text: string; toolCalls: []; steps: [] }>; - }).invokeLLM = async ({ messages }) => { - // System message is passed unchanged through role reversal. - const sysMsg = messages.find((m) => m.role === "system"); - systemPromptCapture = sysMsg?.content ?? ""; + invokeLLM: (p: { instructions?: string }) => Promise<{ text: string; toolCalls: []; steps: [] }>; + }).invokeLLM = async ({ instructions }) => { + // The system prompt rides on `instructions`, outside the messages. + systemPromptCapture = instructions ?? ""; return { text: "I need help please", toolCalls: [], steps: [] }; }; diff --git a/javascript/src/agents/judge/__tests__/judge-agent.test.ts b/javascript/src/agents/judge/__tests__/judge-agent.test.ts index e4828d63c..3177a7723 100644 --- a/javascript/src/agents/judge/__tests__/judge-agent.test.ts +++ b/javascript/src/agents/judge/__tests__/judge-agent.test.ts @@ -218,6 +218,7 @@ describe("JudgeAgent", () => { }); describe("when custom system prompt is used with large trace", () => { + /** @scenario "The judge sends its system prompt as instructions" */ it("preserves the custom system prompt", async () => { const largeTrace = createLargeTrace(); const collector = createMockSpanCollector(largeTrace); @@ -244,15 +245,16 @@ describe("JudgeAgent", () => { await agent.call(createBaseInput()); - const systemMsg = capturedParams!.messages?.find( - (m) => "role" in m && m.role === "system" - ); - expect(systemMsg).toBeDefined(); + const instructions = capturedParams!.instructions; + expect(instructions).toBeDefined(); const content = - typeof systemMsg!.content === "string" - ? systemMsg!.content - : JSON.stringify(systemMsg!.content); + typeof instructions === "string" + ? instructions + : JSON.stringify(instructions); expect(content).toContain(customPrompt); + expect( + capturedParams!.messages?.some((m) => m.role === "system") + ).toBe(false); }); }); diff --git a/javascript/src/agents/judge/__tests__/judge-per-criterion-verdicts.unit.test.ts b/javascript/src/agents/judge/__tests__/judge-per-criterion-verdicts.unit.test.ts index 044910592..f5b8e2405 100644 --- a/javascript/src/agents/judge/__tests__/judge-per-criterion-verdicts.unit.test.ts +++ b/javascript/src/agents/judge/__tests__/judge-per-criterion-verdicts.unit.test.ts @@ -118,8 +118,7 @@ function finishTestSchema(params: InvokeLLMParams): Record { } function systemPrompt(params: InvokeLLMParams): string { - const first = params.messages![0]!; - return typeof first.content === "string" ? first.content : ""; + return typeof params.instructions === "string" ? params.instructions : ""; } class ScriptedAgent extends AgentAdapter { diff --git a/javascript/src/agents/judge/__tests__/judge-provider-matrix.integration.test.ts b/javascript/src/agents/judge/__tests__/judge-provider-matrix.integration.test.ts index 7a60aa58f..29d7078c1 100644 --- a/javascript/src/agents/judge/__tests__/judge-provider-matrix.integration.test.ts +++ b/javascript/src/agents/judge/__tests__/judge-provider-matrix.integration.test.ts @@ -12,7 +12,7 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock"; -import { createGoogleGenerativeAI } from "@ai-sdk/google"; +import { createGoogle } from "@ai-sdk/google"; import { createOpenAI } from "@ai-sdk/openai"; import { loadFeature, describeFeature } from "@amiceli/vitest-cucumber"; import type { LanguageModel, ModelMessage } from "ai"; @@ -58,7 +58,7 @@ const MODELS: Record LanguageModel | null> = { env.OPENAI_API_KEY ? createOpenAI()("gpt-5.6-luna") : null, "gemini gemini-3.5-flash": () => env.GEMINI_API_KEY - ? createGoogleGenerativeAI({ apiKey: env.GEMINI_API_KEY })( + ? createGoogle({ apiKey: env.GEMINI_API_KEY })( "gemini-3.5-flash" ) : null, diff --git a/javascript/src/agents/judge/__tests__/judge-two-phase.unit.test.ts b/javascript/src/agents/judge/__tests__/judge-two-phase.unit.test.ts index 72dad8bb2..422c26efd 100644 --- a/javascript/src/agents/judge/__tests__/judge-two-phase.unit.test.ts +++ b/javascript/src/agents/judge/__tests__/judge-two-phase.unit.test.ts @@ -167,12 +167,7 @@ function schemaShapeKeys(tool: unknown): string[] { } function systemPromptOf(params: InvokeLLMParams | undefined): string { - const systemMessage = params?.messages?.find( - (m) => "role" in m && m.role === "system" - ); - return typeof systemMessage?.content === "string" - ? systemMessage.content - : ""; + return typeof params?.instructions === "string" ? params.instructions : ""; } const feature = await loadFeature(FEATURE_PATH); diff --git a/javascript/src/agents/judge/__tests__/remote-traces.unit.test.ts b/javascript/src/agents/judge/__tests__/remote-traces.unit.test.ts index bea4d48f1..45e2b74ee 100644 --- a/javascript/src/agents/judge/__tests__/remote-traces.unit.test.ts +++ b/javascript/src/agents/judge/__tests__/remote-traces.unit.test.ts @@ -236,13 +236,13 @@ function userMessageContent(params: InvokeLLMParams | undefined): string { } function systemMessageContent(params: InvokeLLMParams | undefined): string { - const systemMessage = params?.messages?.find( - (m) => "role" in m && m.role === "system" - ); - if (!systemMessage) throw new Error("Expected a system message in LLM call"); - return typeof systemMessage.content === "string" - ? systemMessage.content - : JSON.stringify(systemMessage.content); + const instructions = params?.instructions; + if (instructions === undefined) { + throw new Error("Expected instructions in LLM call"); + } + return typeof instructions === "string" + ? instructions + : JSON.stringify(instructions); } // --------------------------------------------------------------------------- diff --git a/javascript/src/agents/judge/judge-agent.ts b/javascript/src/agents/judge/judge-agent.ts index 5cf6d5b35..a75a4c2a2 100644 --- a/javascript/src/agents/judge/judge-agent.ts +++ b/javascript/src/agents/judge/judge-agent.ts @@ -4,7 +4,7 @@ import { ToolSet, Tool, tool, - stepCountIs, + isStepCount, hasToolCall, } from "ai"; import { z } from "zod/v4"; @@ -484,7 +484,7 @@ export class JudgeAgent extends JudgeAgentAdapter { /** * Extract a string identifier from the configured model for substring matching. * - * `LanguageModel` is `GlobalProviderModelId | LanguageModelV3 | LanguageModelV2`. + * `LanguageModel` is `GlobalProviderModelId | LanguageModelV4 | LanguageModelV3 | LanguageModelV2`. * `GlobalProviderModelId` resolves to a string literal; provider objects expose * a `modelId` property. We handle both shapes. */ @@ -492,7 +492,7 @@ export class JudgeAgent extends JudgeAgentAdapter { const model = this.cfg.model; if (!model) return ""; if (typeof model === "string") return model.toLowerCase(); - // LanguageModelV3 / LanguageModelV2 objects expose `modelId`. + // LanguageModelV4 / V3 / V2 objects expose `modelId`. const obj = model as { modelId?: string; provider?: string }; const parts = [obj.provider ?? "", obj.modelId ?? ""].filter(Boolean); return parts.join("/").toLowerCase(); @@ -857,10 +857,8 @@ export class JudgeAgent extends JudgeAgentAdapter { const params: InvokeLLMParams = { model: mergedConfig.model, - messages: [ - { role: "system", content: systemPrompt }, - { role: "user", content: contentForJudge }, - ], + instructions: systemPrompt, + messages: [{ role: "user", content: contentForJudge }], temperature: mergedConfig.temperature, maxOutputTokens: mergedConfig.maxTokens, tools, @@ -868,7 +866,7 @@ export class JudgeAgent extends JudgeAgentAdapter { }; if (isLargeTrace) { params.stopWhen = [ - stepCountIs(this.maxDiscoverySteps), + isStepCount(this.maxDiscoverySteps), hasToolCall("continue_test"), hasToolCall("make_verdict"), ]; @@ -895,8 +893,8 @@ export class JudgeAgent extends JudgeAgentAdapter { ); // generateText never writes back into params.messages, so the discovery // cycles live only on the completion steps. Recombine both before - // collapsing, then drop the two messages the verdict phase rebuilds by - // itself (the system prompt and the criteria block). + // collapsing, then drop the one message the verdict phase rebuilds by + // itself (the criteria block; the system prompt rides on instructions). const discoveryHistory: ModelMessage[] = [ ...(params.messages ?? []), ...(completion.steps ?? []).flatMap((step) => [ @@ -905,7 +903,7 @@ export class JudgeAgent extends JudgeAgentAdapter { ]; return { decision: "exhausted", - discoveryRecap: collapseDiscoveryHistory(discoveryHistory).slice(2), + discoveryRecap: collapseDiscoveryHistory(discoveryHistory).slice(1), }; } return { @@ -1044,7 +1042,6 @@ export class JudgeAgent extends JudgeAgentAdapter { }); const messages: ModelMessage[] = [ - { role: "system", content: systemPrompt }, { role: "user", content: contentForJudge }, ]; @@ -1087,6 +1084,7 @@ export class JudgeAgent extends JudgeAgentAdapter { const params: InvokeLLMParams = { model: mergedConfig.model, + instructions: systemPrompt, messages, temperature: mergedConfig.temperature, maxOutputTokens: mergedConfig.maxTokens, @@ -1098,7 +1096,7 @@ export class JudgeAgent extends JudgeAgentAdapter { if (isLargeTrace && !exhaustedEntry) { params.toolChoice = "required"; params.stopWhen = [ - stepCountIs(this.maxDiscoverySteps), + isStepCount(this.maxDiscoverySteps), hasToolCall("finish_test"), ...(waitExtensionAvailable ? [hasToolCall("wait_for_traces")] : []), ]; @@ -1177,10 +1175,10 @@ export class JudgeAgent extends JudgeAgentAdapter { /** * True when `toolName` was called anywhere in the (possibly multi-step) - * completion — the aggregate `steps` array when present, else the final - * `toolCalls`. AI SDK v6 surfaces only the final step in - * `completion.toolCalls`; a terminal call earlier in the loop would be - * invisible there. + * completion — the aggregate `steps` array when present, else + * `toolCalls`. AI SDK 6 surfaced only the final step in + * `completion.toolCalls` (AI SDK 7 accumulates every step); `steps` reads + * the same under both. */ private completionCalledTool( completion: InvokeLLMResult, diff --git a/javascript/src/agents/llm-invoker.factory.ts b/javascript/src/agents/llm-invoker.factory.ts index 0d0641b44..bde9c61c3 100644 --- a/javascript/src/agents/llm-invoker.factory.ts +++ b/javascript/src/agents/llm-invoker.factory.ts @@ -35,7 +35,7 @@ export const createLLMInvoker = ( try { return await generate({ ...adapted, - experimental_telemetry: { isEnabled: true }, + telemetry: { isEnabled: true }, }); } catch (error) { const adaptation = adaptationFor(error, adapted); diff --git a/javascript/src/agents/red-team/red-team-agent.ts b/javascript/src/agents/red-team/red-team-agent.ts index 2187381fc..a8b05ce30 100644 --- a/javascript/src/agents/red-team/red-team-agent.ts +++ b/javascript/src/agents/red-team/red-team-agent.ts @@ -431,6 +431,9 @@ Reply with exactly this JSON and nothing else: const result = await generateText({ model: this.model, messages: this.attackerHistory as ModelMessage[], + // The prompt plus [SCORE]/[BACKTRACKED]/[INJECTED] markers are timeline + // system turns; on turn 1 the prompt is the only message there is. + allowSystemInMessages: true, temperature: this.temperature, maxOutputTokens: this.maxTokens, }); diff --git a/javascript/src/agents/user-simulator-agent.ts b/javascript/src/agents/user-simulator-agent.ts index ee8f10a62..1831ecccc 100644 --- a/javascript/src/agents/user-simulator-agent.ts +++ b/javascript/src/agents/user-simulator-agent.ts @@ -457,7 +457,6 @@ class UserSimulatorAgent extends UserSimulatorAgentAdapter { const strippedMessages = stripAudioContent(preparedMessages); const messages: ModelMessage[] = [ - { role: "system", content: systemPrompt }, { role: "assistant", content: "Hello, how can I help you today" }, ...strippedMessages, ]; @@ -486,7 +485,10 @@ class UserSimulatorAgent extends UserSimulatorAgentAdapter { ) { const completion = await this.invokeLLM({ model: mergedConfig.model, + instructions: systemPrompt, messages: reversedMessages, + // The transcript may carry system turns a script added with message(). + allowSystemInMessages: true, temperature: mergedConfig.temperature, maxOutputTokens: mergedConfig.maxTokens, }); diff --git a/javascript/src/voice/__tests__/composable-transcript-attach.test.ts b/javascript/src/voice/__tests__/composable-transcript-attach.test.ts index 96d2809a7..1dbc06fd7 100644 --- a/javascript/src/voice/__tests__/composable-transcript-attach.test.ts +++ b/javascript/src/voice/__tests__/composable-transcript-attach.test.ts @@ -37,13 +37,16 @@ const stubStt: STTProvider = { /** Minimal ai-sdk LanguageModel stub returning a fixed reply. */ function stubLlm(text: string): LanguageModel { return { - specificationVersion: "v3" as const, + specificationVersion: "v4" as const, provider: "fake", modelId: "fake-model", supportedUrls: {}, doGenerate: async () => ({ - finishReason: "stop", - usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 1, text: 1, reasoning: 0 }, + }, content: [{ type: "text", text }], warnings: [], }), diff --git a/javascript/src/voice/__tests__/factories.test.ts b/javascript/src/voice/__tests__/factories.test.ts index 0313b935a..8fa070116 100644 --- a/javascript/src/voice/__tests__/factories.test.ts +++ b/javascript/src/voice/__tests__/factories.test.ts @@ -29,13 +29,16 @@ const stubStt: STTProvider = { // Minimal ai-sdk LanguageModel stub — never invoked (factory test only). const stubLlm = { - specificationVersion: "v3" as const, + specificationVersion: "v4" as const, provider: "fake", modelId: "fake-model", supportedUrls: {}, doGenerate: async () => ({ - finishReason: "stop", - usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 1, text: 1, reasoning: 0 }, + }, content: [{ type: "text", text: "ok" }], warnings: [], }), diff --git a/javascript/src/voice/adapters/__tests__/elevenlabs.test.ts b/javascript/src/voice/adapters/__tests__/elevenlabs.test.ts index e02dfd1fc..1b97da529 100644 --- a/javascript/src/voice/adapters/__tests__/elevenlabs.test.ts +++ b/javascript/src/voice/adapters/__tests__/elevenlabs.test.ts @@ -54,8 +54,8 @@ const isUserAudioChunk = (f: Record): boolean => typeof f.user_audio_chunk === "string"; /** - * Hand-rolled LanguageModelV3 stub — we only need `doGenerate`. Cast - * through `unknown` so we don't drag the full LanguageModelV3 type + * Hand-rolled LanguageModelV4 stub — we only need `doGenerate`. Cast + * through `unknown` so we don't drag the full LanguageModelV4 type * surface (and its myriad provider-specific fields) into test code. */ function makeFakeLanguageModel(text: string): { @@ -64,15 +64,18 @@ function makeFakeLanguageModel(text: string): { } { const calls = { current: 0 }; const model = { - specificationVersion: "v3" as const, + specificationVersion: "v4" as const, provider: "fake", modelId: "fake-model", supportedUrls: {}, doGenerate: async () => { calls.current += 1; return { - finishReason: "stop", - usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + finishReason: { unified: "stop", raw: "stop" }, + usage: { + inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 1, text: 1, reasoning: 0 }, + }, content: [{ type: "text", text }], warnings: [], }; diff --git a/javascript/src/voice/adapters/composable.ts b/javascript/src/voice/adapters/composable.ts index 5af9ba5f3..d20caf095 100644 --- a/javascript/src/voice/adapters/composable.ts +++ b/javascript/src/voice/adapters/composable.ts @@ -194,6 +194,9 @@ export class ComposableVoiceAgent extends VoiceAgentAdapter { const result = await generateText({ model: this.llm, messages: this.history, + // history[0] is the system prompt; when the agent opens the call it is + // the only message, and AI SDK 7 refuses empty messages. + allowSystemInMessages: true, }); const responseText = result.text ?? ""; this.lastLlmResponse = responseText; diff --git a/specs/ai-sdk-7-system-prompts.feature b/specs/ai-sdk-7-system-prompts.feature new file mode 100644 index 000000000..3ec185070 --- /dev/null +++ b/specs/ai-sdk-7-system-prompts.feature @@ -0,0 +1,26 @@ +Feature: Testing agents send their system prompts the AI SDK 7 way + As a scenario test author on AI SDK 7 + I want the judge and the user simulator to pass their system prompts as instructions + So that AI SDK 7 does not refuse their calls for carrying system messages + + @unit + Scenario: The user simulator sends its system prompt as instructions + Given a user simulator agent whose model is stubbed + When the simulator produces the user's turn + Then the model call carries the system prompt as instructions + And no message in the model call has the system role + + @unit + Scenario: A system turn added by the script still reaches the user simulator + Given a user simulator agent whose model is stubbed + And the transcript holds a system turn added by the script + When the simulator produces the user's turn + Then the model call allows system messages + And the scripted system turn is among the messages + + @unit + Scenario: The judge sends its system prompt as instructions + Given a judge agent whose model is stubbed + When the judge decides whether to continue + Then the model call carries the system prompt as instructions + And no message in the model call has the system role