diff --git a/.changeset/models-vision-probe-background.md b/.changeset/models-vision-probe-background.md new file mode 100644 index 00000000000..77c18353e76 --- /dev/null +++ b/.changeset/models-vision-probe-background.md @@ -0,0 +1,5 @@ +--- +'@red-hat-developer-hub/backstage-plugin-intelligent-assistant-backend': patch +--- + +Speed up `GET /v1/models` by no longer blocking the response on per-model vision-capability probes. Vision support is served from cache and probed in the background — the cache is warmed at startup and self-heals on cache miss (a model new to the LCS list, or one whose previous probe failed) — so the endpoint returns immediately instead of waiting up to the probe timeout per model. diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts index 2c6499cd09f..b2a0041961d 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts @@ -180,27 +180,79 @@ describe('intelligent-assistant router tests', () => { }); describe('GET v1/models supportsVision enrichment', () => { + // Polls until the predicate is truthy; keeps the non-blocking background + // probe tests deterministic without a fixed sleep. + const waitFor = async ( + predicate: () => boolean | Promise, + { tries = 50, delayMs = 10 } = {}, + ): Promise => { + for (let i = 0; i < tries; i += 1) { + if (await predicate()) return; + await new Promise(resolve => setTimeout(resolve, delayMs)); + } + throw new Error('waitFor: condition not met in time'); + }; + beforeEach(() => { ModelCapabilitiesCache.clear(); }); - it('enriches each model with supportsVision:true when the vision probe succeeds', async () => { + it('lazily probes a model absent from the warm cache (LCS list changed)', async () => { server.use( http.post(`${LOCAL_LCS_ADDR}/v1/responses`, () => HttpResponse.json({ id: 'resp-1', output: [] }), ), ); + // Startup warm-up probes the default models; wait until it settles. const backendServer = await startBackendServer(); - const response = await request(backendServer).get( + await waitFor( + () => + ModelCapabilitiesCache.get('openai/gpt-4-turbo') === true && + ModelCapabilitiesCache.get('team-cluster/qwen25-7b-instruct') === + true, + ); + + // A new model appears in the LCS list that the warm-up never saw. + server.use( + http.get(`${LOCAL_LCS_ADDR}/v1/models`, () => + HttpResponse.json({ + models: [ + { + identifier: 'openai/gpt-4-turbo', + api_model_type: 'llm', + }, + { + identifier: 'openai/gpt-new-vision', + api_model_type: 'llm', + }, + ], + }), + ), + ); + + // The un-warmed model is a cache miss: returned with the conservative + // default now, probed off the request path. + const first = await request(backendServer).get( '/api/intelligent-assistant/v1/models', ); + expect(first.status).toBe(200); + const firstNew = first.body.models.find( + (m: any) => m.identifier === 'openai/gpt-new-vision', + ); + expect(firstNew.supportsVision).toBe(false); - expect(response.status).toBe(200); - expect(response.body.models).toHaveLength(2); - for (const model of response.body.models) { - expect(model.supportsVision).toBe(true); - } + // Background probe warms it, so a later call reports the real value. + await waitFor( + () => ModelCapabilitiesCache.get('openai/gpt-new-vision') === true, + ); + const second = await request(backendServer).get( + '/api/intelligent-assistant/v1/models', + ); + const secondNew = second.body.models.find( + (m: any) => m.identifier === 'openai/gpt-new-vision', + ); + expect(secondNew.supportsVision).toBe(true); }); it('sets supportsVision:false when the vision probe returns a non-ok response', async () => { @@ -211,18 +263,24 @@ describe('intelligent-assistant router tests', () => { ), ); + // Warm-up probes both default models; a non-ok probe caches false. const backendServer = await startBackendServer(); + await waitFor( + () => + ModelCapabilitiesCache.has('openai/gpt-4-turbo') && + ModelCapabilitiesCache.has('team-cluster/qwen25-7b-instruct'), + ); + const response = await request(backendServer).get( '/api/intelligent-assistant/v1/models', ); - expect(response.status).toBe(200); for (const model of response.body.models) { expect(model.supportsVision).toBe(false); } }); - it('sets supportsVision:false when the vision probe errors (network/timeout)', async () => { + it('does not cache a probe that errors (network/timeout)', async () => { server.use( http.post(`${LOCAL_LCS_ADDR}/v1/responses`, () => HttpResponse.error()), ); @@ -262,15 +320,19 @@ describe('intelligent-assistant router tests', () => { }); it('reuses the cache and does not re-probe an already-validated model', async () => { - // Cache key is the model identifier used directly. + // Pre-seed one model so the startup warm-up skips it. ModelCapabilitiesCache.set('openai/gpt-4-turbo', true); - let probeCount = 0; + const probedKeys: string[] = []; server.use( - http.post(`${LOCAL_LCS_ADDR}/v1/responses`, () => { - probeCount += 1; - return HttpResponse.json({ id: 'resp-1', output: [] }); - }), + http.post( + `${LOCAL_LCS_ADDR}/v1/responses`, + async ({ request: req }) => { + const body = (await req.json()) as { model: string }; + probedKeys.push(body.model); + return HttpResponse.json({ id: 'resp-1', output: [] }); + }, + ), ); const backendServer = await startBackendServer(); @@ -283,8 +345,11 @@ describe('intelligent-assistant router tests', () => { (m: any) => m.identifier === 'openai/gpt-4-turbo', ); expect(cached.supportsVision).toBe(true); - // Only the second, uncached model should have triggered a probe. - expect(probeCount).toBe(1); + // Only the un-seeded model should ever be probed; gpt-4-turbo never is. + await waitFor(() => + probedKeys.includes('team-cluster/qwen25-7b-instruct'), + ); + expect(probedKeys).not.toContain('openai/gpt-4-turbo'); }); it('does not probe non-llm (e.g. embedding) models and marks them supportsVision:false', async () => { @@ -333,9 +398,44 @@ describe('intelligent-assistant router tests', () => { (m: any) => m.api_model_type === 'llm', ); expect(embedding.supportsVision).toBe(false); - expect(llm.supportsVision).toBe(true); - // Only the llm model should have been probed. + // The startup warm-up probes only the llm, so it is already enriched. + await waitFor(() => probeCount === 1); + const warmed = await request(backendServer).get( + '/api/intelligent-assistant/v1/models', + ); + const warmedLlm = warmed.body.models.find( + (m: any) => m.api_model_type === 'llm', + ); + expect(warmedLlm.supportsVision).toBe(true); + // The embedding is never probed. expect(probeCount).toBe(1); + expect(llm.identifier).toBe('openai/gpt-4-turbo'); + }); + + it('warms the vision cache at startup so the first request is already enriched', async () => { + server.use( + http.post(`${LOCAL_LCS_ADDR}/v1/responses`, () => + HttpResponse.json({ id: 'resp-1', output: [] }), + ), + ); + + // Creating the router kicks off the startup warm-up; the cache fills + // before any /v1/models request is made. + const backendServer = await startBackendServer(); + await waitFor( + () => + ModelCapabilitiesCache.get('openai/gpt-4-turbo') === true && + ModelCapabilitiesCache.get('team-cluster/qwen25-7b-instruct') === + true, + ); + + const response = await request(backendServer).get( + '/api/intelligent-assistant/v1/models', + ); + expect(response.status).toBe(200); + for (const model of response.body.models) { + expect(model.supportsVision).toBe(true); + } }); }); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.ts index 094ab480348..3391d89ba47 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.ts @@ -230,6 +230,64 @@ export async function createRouter( ); const lcsBaseUrl = `http://${DEFAULT_LIGHTSPEED_SERVICE_HOST}:${port}`; + // Vision probes currently in flight, so a burst of /v1/models requests + // triggers at most one background probe per model. Warming the cache off the + // request path keeps /v1/models fast even when a probe hits its timeout. + const inFlightVisionProbes = new Set(); + const probeModelVisionInBackground = (cacheKey: string): void => { + if (inFlightVisionProbes.has(cacheKey)) return; + inFlightVisionProbes.add(cacheKey); + probeModelVisionSupport(lcsBaseUrl, cacheKey) + .catch(error => { + logger.warn( + `Background vision probe failed for ${cacheKey}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + }) + .finally(() => { + inFlightVisionProbes.delete(cacheKey); + }); + }; + + // Pre-warms the vision cache by fetching the current LCS model list and + // probing any llm not already cached. Runs at startup and self-heals: because + // it only probes cache misses, it re-probes models that are new in the LCS + // list or whose previous probe failed and was not cached, while leaving + // already-known models untouched. + const warmModelVisionCache = async (): Promise => { + const upstream = await fetch(`${lcsBaseUrl}/v1/models`); + if (!upstream.ok) { + logger.warn( + `Vision cache warm-up skipped: LCS /v1/models returned ${upstream.status}`, + ); + return; + } + const data = (await upstream.json()) as { + models?: Array<{ identifier: string; api_model_type?: string }>; + }; + const models = Array.isArray(data.models) ? data.models : []; + for (const model of models) { + if ( + model.api_model_type === 'llm' && + !ModelCapabilitiesCache.has(model.identifier) + ) { + probeModelVisionInBackground(model.identifier); + } + } + }; + + // Warm at load so the first /v1/models request is already enriched. Fire-and- + // forget: LCS may not be reachable yet, and a cold cache still self-heals via + // the per-request background probe on cache miss. + warmModelVisionCache().catch(error => { + logger.warn( + `Vision cache warm-up failed: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + }); + const apiProxy = createProxyMiddleware({ target: lcsBaseUrl, changeOrigin: true, @@ -685,32 +743,32 @@ export async function createRouter( }; const models = Array.isArray(data.models) ? data.models : []; - const enriched = await Promise.all( - models.map(async model => { - // Only LLMs can be vision-capable; skip probing embeddings etc. - if (model.api_model_type !== 'llm') { - return { ...model, supportsVision: false }; - } - // `identifier` is already the `provider/model` key LCS and - // /v1/validate-model-vision use — do not re-prefix with provider_id. - const cacheKey = model.identifier; - try { - const supportsVision = await probeModelVisionSupport( - lcsBaseUrl, - cacheKey, - ); - return { ...model, supportsVision }; - } catch (error) { - // One failed probe must not fail the whole list. - logger.warn( - `Vision probe failed for ${cacheKey}: ${ - error instanceof Error ? error.message : String(error) - }`, - ); - return { ...model, supportsVision: false }; - } - }), - ); + const enriched = models.map(model => { + // Only LLMs can be vision-capable; skip probing embeddings etc. + if (model.api_model_type !== 'llm') { + return { ...model, supportsVision: false }; + } + // `identifier` is already the `provider/model` key LCS and + const cacheKey = model.identifier; + const cached = ModelCapabilitiesCache.get(cacheKey); + if (cached !== undefined) { + console.log('vision probe success', { + ...model, + supportsVision: cached, + }); + return { ...model, supportsVision: cached }; + } + // Cache miss: never block the list on a probe (up to + // VISION_PROBE_TIMEOUT_MS each). Report the conservative default now + // and warm the cache in the background so the next /v1/models call + // returns the real value. + probeModelVisionInBackground(cacheKey); + console.log('vision probe failed', { + ...model, + supportsVision: false, + }); + return { ...model, supportsVision: false }; + }); response.json({ ...data, models: enriched }); } catch (error) {