From 1f6309e92494b10efa2fb64b03ac10a5c8ba7151 Mon Sep 17 00:00:00 2001 From: ClaudiaFang Date: Wed, 28 Oct 2026 10:59:35 +0800 Subject: [PATCH 1/2] fix(analysis): avoid false rage loops from empty tools and invalid timing (#34) --- .../analysis/__tests__/loop-detector.test.ts | 29 +++++++++++++++++-- cli/src/analysis/loop-detector.ts | 19 ++++++++---- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/cli/src/analysis/__tests__/loop-detector.test.ts b/cli/src/analysis/__tests__/loop-detector.test.ts index 66d72357..62d57bed 100644 --- a/cli/src/analysis/__tests__/loop-detector.test.ts +++ b/cli/src/analysis/__tests__/loop-detector.test.ts @@ -25,9 +25,10 @@ describe('detectRageLoopHeuristic', () => { mockMessage('5', '2026-04-24T10:00:40Z', JSON.stringify({ inputTokens: 150000 })), ]; - // Add semantic repetition - messages[3].content = 'repeat content'; - messages[4].content = 'repeat content'; + // Repeat a substantive failure; empty [] tool-call fields are not evidence. + messages[2].content = 'Timeout when deploying exactly the same revision'; + messages[3].content = 'Timeout when deploying exactly the same revision'; + messages[4].content = 'Timeout when deploying exactly the same revision'; const result = detectRageLoopHeuristic(messages); expect(result.detected).toBe(true); @@ -35,6 +36,28 @@ describe('detectRageLoopHeuristic', () => { expect(result.turnRange).toEqual(['Turn#0', 'Turn#4']); }); + it('does not classify empty tool calls and short, generic responses as a rage loop', () => { + const messages = Array.from({ length: 5 }, (_, i) => + mockMessage(String(i), `2026-04-24T10:00:${String(i * 10).padStart(2, '0')}Z`, JSON.stringify({ inputTokens: 150000 }), 'OK') + ); + expect(detectRageLoopHeuristic(messages).detected).toBe(false); + }); + + it('does not treat out-of-order timestamps as a rapid loop', () => { + const messages = Array.from({ length: 5 }, (_, i) => + mockMessage(String(i), `2026-04-24T10:00:${String(i * 10).padStart(2, '0')}Z`, JSON.stringify({ inputTokens: 150000 }), 'Timeout when deploying exactly the same revision') + ); + messages[2].timestamp = '2026-04-24T09:00:00Z'; + expect(detectRageLoopHeuristic(messages).detected).toBe(false); + }); + + it('classifies repeated low-context errors separately from rage loops', () => { + const messages = Array.from({ length: 5 }, (_, i) => + mockMessage(String(i), `2026-04-24T10:00:${String(i * 10).padStart(2, '0')}Z`, JSON.stringify({ inputTokens: 8000 }), 'Timeout when deploying exactly the same revision') + ); + expect(detectRageLoopHeuristic(messages).detected).toBe(false); + }); + it('should not detect a loop if token count is growing', () => { const messages: SQLiteMessageRow[] = [ mockMessage('1', '2026-04-24T10:00:00Z', JSON.stringify({ inputTokens: 100000 })), diff --git a/cli/src/analysis/loop-detector.ts b/cli/src/analysis/loop-detector.ts index 4ff49ea4..3af4bc49 100644 --- a/cli/src/analysis/loop-detector.ts +++ b/cli/src/analysis/loop-detector.ts @@ -29,7 +29,7 @@ export function detectRageLoopHeuristic(messages: SQLiteMessageRow[]): RageLoopS if (prevMsg) { const delta = (new Date(msg.timestamp).getTime() - new Date(prevMsg.timestamp).getTime()) / 1000; - if (delta < 60) { + if (Number.isFinite(delta) && delta >= 0 && delta < 60) { currentCluster.push(msg); } else { if (currentCluster.length >= 4) clusters.push([...currentCluster]); @@ -46,7 +46,7 @@ export function detectRageLoopHeuristic(messages: SQLiteMessageRow[]): RageLoopS const tokens = cluster.map(m => { try { const usage = JSON.parse(m.usage || '{}'); - return usage.inputTokens || usage.totalTokens || 0; + return usage.inputTokens || usage.input_tokens || usage.totalTokens || usage.total_tokens || 0; } catch { return 0; } @@ -63,8 +63,15 @@ export function detectRageLoopHeuristic(messages: SQLiteMessageRow[]): RageLoopS const repetitions = cluster.filter((m, i) => { if (i === 0) return false; const prev = cluster[i - 1]; - return m.content.slice(0, 100) === prev.content.slice(0, 100) || - m.tool_calls === prev.tool_calls; + // Empty tool-call arrays are common and cannot demonstrate repetition. + const content = m.content.trim().replace(/\s+/g, ' ').slice(0, 120); + const prevContent = prev.content.trim().replace(/\s+/g, ' ').slice(0, 120); + const sameContent = content.length >= 12 && content === prevContent; + // Only compare actual tool invocations, not null or [] placeholders. + const calls = m.tool_calls?.trim(); + const prevCalls = prev.tool_calls?.trim(); + const sameCalls = Boolean(calls && calls !== '[]' && calls === prevCalls); + return sameContent || sameCalls; }); if (repetitions.length >= 2) { @@ -73,8 +80,8 @@ export function detectRageLoopHeuristic(messages: SQLiteMessageRow[]): RageLoopS return { detected: true, - confidence: 0.85, - reasoning: `Detected ${cluster.length} messages in ${Math.round((new Date(cluster[cluster.length-1].timestamp).getTime() - new Date(cluster[0].timestamp).getTime())/1000)}s with static token count (${Math.round(avg/1000)}k) and repetitive signals.`, + confidence: 0.8, + reasoning: `Suspected loop (verify stalled progress): ${cluster.length} messages in ${Math.round((new Date(cluster[cluster.length-1].timestamp).getTime() - new Date(cluster[0].timestamp).getTime())/1000)}s with static token count (${Math.round(avg/1000)}k) and repetitive signals.`, turnRange: [`Turn#${firstTurn}`, `Turn#${lastTurn}`] }; } From 092d7c6caaf78f59fcdf12cff90b73f518e0cf2c Mon Sep 17 00:00:00 2001 From: ClaudiaFang Date: Wed, 28 Oct 2026 10:59:58 +0800 Subject: [PATCH 2/2] fix(analysis): align rage-loop JSON schema and wire server/facet analysis hints (#34) --- cli/package.json | 1 + cli/src/analysis/__tests__/prompts.test.ts | 8 ++++++++ cli/src/analysis/schemas/session-analysis.json | 1 + server/src/llm/analysis.ts | 8 +++++--- server/src/llm/facet-extraction.ts | 3 ++- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/cli/package.json b/cli/package.json index 7ec740b9..e374e6fd 100644 --- a/cli/package.json +++ b/cli/package.json @@ -20,6 +20,7 @@ "./utils/pricing": "./dist/utils/pricing.js", "./utils/telemetry": "./dist/utils/telemetry.js", "./analysis/prompts": "./dist/analysis/prompts.js", + "./analysis/loop-detector": "./dist/analysis/loop-detector.js", "./analysis/prompt-types": "./dist/analysis/prompt-types.js", "./analysis/prompt-constants": "./dist/analysis/prompt-constants.js", "./analysis/message-format": "./dist/analysis/message-format.js", diff --git a/cli/src/analysis/__tests__/prompts.test.ts b/cli/src/analysis/__tests__/prompts.test.ts index 6ff784bd..fcc142d9 100644 --- a/cli/src/analysis/__tests__/prompts.test.ts +++ b/cli/src/analysis/__tests__/prompts.test.ts @@ -384,6 +384,14 @@ describe('buildCacheableConversationBlock', () => { // ────────────────────────────────────────────────────── describe('buildSessionAnalysisInstructions', () => { + it('emits an advisory rage-loop hint only when detector reports a suspected loop', () => { + const signal = { detected: true, confidence: 0.8, reasoning: 'Repeated commands in static context', turnRange: ['Turn#0', 'Turn#4'] as [string, string] }; + const prompted = buildSessionAnalysisInstructions('proj', null, undefined, signal); + expect(prompted).toContain(''); + expect(prompted).toContain('Turn#0 to Turn#4'); + expect(buildSessionAnalysisInstructions('proj', null)).not.toContain(''); + }); + it('includes project name in the instructions', () => { const result = buildSessionAnalysisInstructions('my-app', null); expect(result).toContain('my-app'); diff --git a/cli/src/analysis/schemas/session-analysis.json b/cli/src/analysis/schemas/session-analysis.json index 74bebf83..2dabdf1c 100644 --- a/cli/src/analysis/schemas/session-analysis.json +++ b/cli/src/analysis/schemas/session-analysis.json @@ -61,6 +61,7 @@ "context-loss", "scope-creep", "repeated-mistakes", + "rage-loop", "documentation-gap", "tooling-limitation" ] diff --git a/server/src/llm/analysis.ts b/server/src/llm/analysis.ts index 800e01fb..42c3ac81 100644 --- a/server/src/llm/analysis.ts +++ b/server/src/llm/analysis.ts @@ -44,6 +44,7 @@ import { import { calculateAnalysisCost } from './analysis-pricing.js'; import { saveAnalysisUsage } from './analysis-usage-db.js'; import { getDb } from '@code-insights/cli/db/client'; +import { detectRageLoopHeuristic } from '@code-insights/cli/analysis/loop-detector'; import { loadConfig } from '@code-insights/cli/utils/config'; import * as sqliteVec from 'sqlite-vec'; import { embedOne, DEFAULT_EMBEDDING_CONFIG } from '@code-insights/cli/embeddings/client'; @@ -93,6 +94,7 @@ export async function analyzeSession( const formattedMessages = formatMessagesForAnalysis(messages); const estimatedTokens = client.estimateTokens(formattedMessages); const sessionMeta = buildSessionMeta(session); + const loopSignal = detectRageLoopHeuristic(messages); // Retrieve related insights for RAG context (AutoRefine pattern) const retrievalConfig = getRetrievalConfig(); @@ -129,7 +131,7 @@ export async function analyzeSession( { role: 'system', content: SHARED_ANALYST_SYSTEM_PROMPT }, { role: 'user', content: [ buildCacheableConversationBlock(chunkFormatted), - { type: 'text' as const, text: buildSessionAnalysisInstructions(session.project_name, session.summary, sessionMeta, undefined, relatedInsights) }, + { type: 'text' as const, text: buildSessionAnalysisInstructions(session.project_name, session.summary, sessionMeta, detectRageLoopHeuristic(chunk), relatedInsights) }, ] }, ], { signal: options?.signal }); @@ -171,7 +173,7 @@ export async function analyzeSession( { role: 'system', content: SHARED_ANALYST_SYSTEM_PROMPT }, { role: 'user', content: [ buildCacheableConversationBlock(facetMessages), - { type: 'text' as const, text: buildFacetOnlyInstructions(session.project_name, session.summary, sessionMeta, undefined, relatedInsights) }, + { type: 'text' as const, text: buildFacetOnlyInstructions(session.project_name, session.summary, sessionMeta, loopSignal, relatedInsights) }, ] }, ], { signal: options?.signal }); @@ -205,7 +207,7 @@ export async function analyzeSession( { role: 'system', content: SHARED_ANALYST_SYSTEM_PROMPT }, { role: 'user', content: [ buildCacheableConversationBlock(formattedMessages), - { type: 'text' as const, text: buildSessionAnalysisInstructions(session.project_name, session.summary, sessionMeta, undefined, relatedInsights) }, + { type: 'text' as const, text: buildSessionAnalysisInstructions(session.project_name, session.summary, sessionMeta, loopSignal, relatedInsights) }, ] }, ], { signal: options?.signal }); diff --git a/server/src/llm/facet-extraction.ts b/server/src/llm/facet-extraction.ts index a6a8ea0e..272fefd6 100644 --- a/server/src/llm/facet-extraction.ts +++ b/server/src/llm/facet-extraction.ts @@ -15,6 +15,7 @@ import { type SessionData, } from './analysis-db.js'; import { getMaxInputTokens, buildSessionMeta } from './analysis-internal.js'; +import { detectRageLoopHeuristic } from '@code-insights/cli/analysis/loop-detector'; /** * Extract facets only for a session that already has insights (backfill). @@ -52,7 +53,7 @@ export async function extractFacetsOnly( { role: 'system', content: SHARED_ANALYST_SYSTEM_PROMPT }, { role: 'user', content: [ buildCacheableConversationBlock(formattedMessages), - { type: 'text' as const, text: buildFacetOnlyInstructions(session.project_name, session.summary, sessionMeta) }, + { type: 'text' as const, text: buildFacetOnlyInstructions(session.project_name, session.summary, sessionMeta, detectRageLoopHeuristic(messages)) }, ] }, ], { signal: options?.signal });