perf(intelligent-assistant): make GET /v1/models non-blocking

.changeset/models-vision-probe-background.md

@@ -0,0 +1,5 @@
+---
+'@red-hat-developer-hub/backstage-plugin-intelligent-assistant-backend': patch
+---
+
+Speed up `GET /v1/models` by no longer blocking the response on per-model vision-capability probes. Vision support is served from cache and probed in the background — the cache is warmed at startup and self-heals on cache miss (a model new to the LCS list, or one whose previous probe failed) — so the endpoint returns immediately instead of waiting up to the probe timeout per model.

workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts

@@ -180,27 +180,79 @@ describe('intelligent-assistant router tests', () => {
   });
 
   describe('GET v1/models supportsVision enrichment', () => {
+    // Polls until the predicate is truthy; keeps the non-blocking background
+    // probe tests deterministic without a fixed sleep.
+    const waitFor = async (
+      predicate: () => boolean | Promise<boolean>,
+      { tries = 50, delayMs = 10 } = {},
+    ): Promise<void> => {
+      for (let i = 0; i < tries; i += 1) {
+        if (await predicate()) return;
+        await new Promise(resolve => setTimeout(resolve, delayMs));
+      }
+      throw new Error('waitFor: condition not met in time');
+    };
+
     beforeEach(() => {
       ModelCapabilitiesCache.clear();
     });
 
-    it('enriches each model with supportsVision:true when the vision probe succeeds', async () => {
+    it('lazily probes a model absent from the warm cache (LCS list changed)', async () => {
       server.use(
         http.post(`${LOCAL_LCS_ADDR}/v1/responses`, () =>
           HttpResponse.json({ id: 'resp-1', output: [] }),
         ),
       );
 
+      // Startup warm-up probes the default models; wait until it settles.
       const backendServer = await startBackendServer();
-      const response = await request(backendServer).get(
+      await waitFor(
+        () =>
+          ModelCapabilitiesCache.get('openai/gpt-4-turbo') === true &&
+          ModelCapabilitiesCache.get('team-cluster/qwen25-7b-instruct') ===
+            true,
+      );
+
+      // A new model appears in the LCS list that the warm-up never saw.
+      server.use(
+        http.get(`${LOCAL_LCS_ADDR}/v1/models`, () =>
+          HttpResponse.json({
+            models: [
+              {
+                identifier: 'openai/gpt-4-turbo',
+                api_model_type: 'llm',
+              },
+              {
+                identifier: 'openai/gpt-new-vision',
+                api_model_type: 'llm',
+              },
+            ],
+          }),
+        ),
+      );
+
+      // The un-warmed model is a cache miss: returned with the conservative
+      // default now, probed off the request path.
+      const first = await request(backendServer).get(
         '/api/intelligent-assistant/v1/models',
       );
+      expect(first.status).toBe(200);
+      const firstNew = first.body.models.find(
+        (m: any) => m.identifier === 'openai/gpt-new-vision',
+      );
+      expect(firstNew.supportsVision).toBe(false);
 
-      expect(response.status).toBe(200);
-      expect(response.body.models).toHaveLength(2);
-      for (const model of response.body.models) {
-        expect(model.supportsVision).toBe(true);
-      }
+      // Background probe warms it, so a later call reports the real value.
+      await waitFor(
+        () => ModelCapabilitiesCache.get('openai/gpt-new-vision') === true,
+      );
+      const second = await request(backendServer).get(
+        '/api/intelligent-assistant/v1/models',
+      );
+      const secondNew = second.body.models.find(
+        (m: any) => m.identifier === 'openai/gpt-new-vision',
+      );
+      expect(secondNew.supportsVision).toBe(true);
     });
 
     it('sets supportsVision:false when the vision probe returns a non-ok response', async () => {
@@ -211,18 +263,24 @@ describe('intelligent-assistant router tests', () => {
         ),
       );
 
+      // Warm-up probes both default models; a non-ok probe caches false.
       const backendServer = await startBackendServer();
+      await waitFor(
+        () =>
+          ModelCapabilitiesCache.has('openai/gpt-4-turbo') &&
+          ModelCapabilitiesCache.has('team-cluster/qwen25-7b-instruct'),
+      );
+
       const response = await request(backendServer).get(
         '/api/intelligent-assistant/v1/models',
       );
-
       expect(response.status).toBe(200);
       for (const model of response.body.models) {
         expect(model.supportsVision).toBe(false);
       }
     });
 
-    it('sets supportsVision:false when the vision probe errors (network/timeout)', async () => {
+    it('does not cache a probe that errors (network/timeout)', async () => {
       server.use(
         http.post(`${LOCAL_LCS_ADDR}/v1/responses`, () => HttpResponse.error()),
       );
@@ -262,15 +320,19 @@ describe('intelligent-assistant router tests', () => {
     });
 
     it('reuses the cache and does not re-probe an already-validated model', async () => {
-      // Cache key is the model identifier used directly.
+      // Pre-seed one model so the startup warm-up skips it.
       ModelCapabilitiesCache.set('openai/gpt-4-turbo', true);
 
-      let probeCount = 0;
+      const probedKeys: string[] = [];
       server.use(
-        http.post(`${LOCAL_LCS_ADDR}/v1/responses`, () => {
-          probeCount += 1;
-          return HttpResponse.json({ id: 'resp-1', output: [] });
-        }),
+        http.post(
+          `${LOCAL_LCS_ADDR}/v1/responses`,
+          async ({ request: req }) => {
+            const body = (await req.json()) as { model: string };
+            probedKeys.push(body.model);
+            return HttpResponse.json({ id: 'resp-1', output: [] });
+          },
+        ),
       );
 
       const backendServer = await startBackendServer();
@@ -283,8 +345,11 @@ describe('intelligent-assistant router tests', () => {
         (m: any) => m.identifier === 'openai/gpt-4-turbo',
       );
       expect(cached.supportsVision).toBe(true);
-      // Only the second, uncached model should have triggered a probe.
-      expect(probeCount).toBe(1);
+      // Only the un-seeded model should ever be probed; gpt-4-turbo never is.
+      await waitFor(() =>
+        probedKeys.includes('team-cluster/qwen25-7b-instruct'),
+      );
+      expect(probedKeys).not.toContain('openai/gpt-4-turbo');
     });
 
     it('does not probe non-llm (e.g. embedding) models and marks them supportsVision:false', async () => {
@@ -333,9 +398,44 @@ describe('intelligent-assistant router tests', () => {
         (m: any) => m.api_model_type === 'llm',
       );
       expect(embedding.supportsVision).toBe(false);
-      expect(llm.supportsVision).toBe(true);
-      // Only the llm model should have been probed.
+      // The startup warm-up probes only the llm, so it is already enriched.
+      await waitFor(() => probeCount === 1);
+      const warmed = await request(backendServer).get(
+        '/api/intelligent-assistant/v1/models',
+      );
+      const warmedLlm = warmed.body.models.find(
+        (m: any) => m.api_model_type === 'llm',
+      );
+      expect(warmedLlm.supportsVision).toBe(true);
+      // The embedding is never probed.
       expect(probeCount).toBe(1);
+      expect(llm.identifier).toBe('openai/gpt-4-turbo');
+    });
+
+    it('warms the vision cache at startup so the first request is already enriched', async () => {
+      server.use(
+        http.post(`${LOCAL_LCS_ADDR}/v1/responses`, () =>
+          HttpResponse.json({ id: 'resp-1', output: [] }),
+        ),
+      );
+
+      // Creating the router kicks off the startup warm-up; the cache fills
+      // before any /v1/models request is made.
+      const backendServer = await startBackendServer();
+      await waitFor(
+        () =>
+          ModelCapabilitiesCache.get('openai/gpt-4-turbo') === true &&
+          ModelCapabilitiesCache.get('team-cluster/qwen25-7b-instruct') ===
+            true,
+      );
+
+      const response = await request(backendServer).get(
+        '/api/intelligent-assistant/v1/models',
+      );
+      expect(response.status).toBe(200);
+      for (const model of response.body.models) {
+        expect(model.supportsVision).toBe(true);
+      }
     });
   });
 

workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.ts

@@ -230,6 +230,64 @@ export async function createRouter(
   );
   const lcsBaseUrl = `http://${DEFAULT_LIGHTSPEED_SERVICE_HOST}:${port}`;
 
+  // Vision probes currently in flight, so a burst of /v1/models requests
+  // triggers at most one background probe per model. Warming the cache off the
+  // request path keeps /v1/models fast even when a probe hits its timeout.
+  const inFlightVisionProbes = new Set<string>();
+  const probeModelVisionInBackground = (cacheKey: string): void => {
+    if (inFlightVisionProbes.has(cacheKey)) return;
+    inFlightVisionProbes.add(cacheKey);
+    probeModelVisionSupport(lcsBaseUrl, cacheKey)
+      .catch(error => {
+        logger.warn(
+          `Background vision probe failed for ${cacheKey}: ${
+            error instanceof Error ? error.message : String(error)
+          }`,
+        );
+      })
+      .finally(() => {
+        inFlightVisionProbes.delete(cacheKey);
+      });
+  };
+
+  // Pre-warms the vision cache by fetching the current LCS model list and
+  // probing any llm not already cached. Runs at startup and self-heals: because
+  // it only probes cache misses, it re-probes models that are new in the LCS
+  // list or whose previous probe failed and was not cached, while leaving
+  // already-known models untouched.
+  const warmModelVisionCache = async (): Promise<void> => {
+    const upstream = await fetch(`${lcsBaseUrl}/v1/models`);
+    if (!upstream.ok) {
+      logger.warn(
+        `Vision cache warm-up skipped: LCS /v1/models returned ${upstream.status}`,
+      );
+      return;
+    }
+    const data = (await upstream.json()) as {
+      models?: Array<{ identifier: string; api_model_type?: string }>;
+    };
+    const models = Array.isArray(data.models) ? data.models : [];
+    for (const model of models) {
+      if (
+        model.api_model_type === 'llm' &&
+        !ModelCapabilitiesCache.has(model.identifier)
+      ) {
+        probeModelVisionInBackground(model.identifier);
+      }
+    }
+  };
+
+  // Warm at load so the first /v1/models request is already enriched. Fire-and-
+  // forget: LCS may not be reachable yet, and a cold cache still self-heals via
+  // the per-request background probe on cache miss.
+  warmModelVisionCache().catch(error => {
+    logger.warn(
+      `Vision cache warm-up failed: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+    );
+  });
+
   const apiProxy = createProxyMiddleware({
     target: lcsBaseUrl,
     changeOrigin: true,
@@ -685,32 +743,32 @@ export async function createRouter(
         };
         const models = Array.isArray(data.models) ? data.models : [];
 
-        const enriched = await Promise.all(
-          models.map(async model => {
-            // Only LLMs can be vision-capable; skip probing embeddings etc.
-            if (model.api_model_type !== 'llm') {
-              return { ...model, supportsVision: false };
-            }
-            // `identifier` is already the `provider/model` key LCS and
-            // /v1/validate-model-vision use — do not re-prefix with provider_id.
-            const cacheKey = model.identifier;
-            try {
-              const supportsVision = await probeModelVisionSupport(
-                lcsBaseUrl,
-                cacheKey,
-              );
-              return { ...model, supportsVision };
-            } catch (error) {
-              // One failed probe must not fail the whole list.
-              logger.warn(
-                `Vision probe failed for ${cacheKey}: ${
-                  error instanceof Error ? error.message : String(error)
-                }`,
-              );
-              return { ...model, supportsVision: false };
-            }
-          }),
-        );
+        const enriched = models.map(model => {
+          // Only LLMs can be vision-capable; skip probing embeddings etc.
+          if (model.api_model_type !== 'llm') {
+            return { ...model, supportsVision: false };
+          }
+          // `identifier` is already the `provider/model` key LCS and
+          const cacheKey = model.identifier;
+          const cached = ModelCapabilitiesCache.get(cacheKey);
+          if (cached !== undefined) {
+            console.log('vision probe success', {
+              ...model,
+              supportsVision: cached,
+            });
+            return { ...model, supportsVision: cached };
+          }
+          // Cache miss: never block the list on a probe (up to
+          // VISION_PROBE_TIMEOUT_MS each). Report the conservative default now
+          // and warm the cache in the background so the next /v1/models call
+          // returns the real value.
+          probeModelVisionInBackground(cacheKey);
+          console.log('vision probe failed', {
+            ...model,
+            supportsVision: false,
+          });
+          return { ...model, supportsVision: false };
+        });
 
         response.json({ ...data, models: enriched });
       } catch (error) {