diff --git a/README.md b/README.md index b891e70..63afdfb 100644 --- a/README.md +++ b/README.md @@ -248,7 +248,7 @@ sequenceDiagram Confluence retrieval supports a configured backend-side Atlassian REST connection. Jira, GitHub, and Slack records are retrieved through configured backend connector endpoints and normalized by the backend; the ingestion pipeline verifies provenance before accepting source records into the company graph. Source-library routes read persisted graph records rather than calling providers from the browser. -`GET /api/integrations/status` reports configuration, indexed counts, and recorded sync outcomes. It does not actively verify upstream credentials or connectivity. A configured endpoint or credential alone does not establish a healthy live connection. +`GET /api/integrations/status` performs a bounded sample fetch for each configured source, reports cumulative distinct indexed counts, and returns the latest recorded sync outcome. Sample reachability does not establish a successful ingestion run. Confluence is the live provider; Jira, Slack, and GitHub currently use the seeded mock API. ## Screenshots image image @@ -284,9 +284,10 @@ The backend currently exposes Express routes directly; it does not publish an Op | `GET /api/graph/neighborhood/:entityType/:entityId` | Bounded direct neighborhood with relationship direction. | | `GET /api/search?q=...` | Up to 50 name/title/email matches. | | `GET /api/documents`, `/api/tickets`, `/api/conversations`, `/api/proactive-insights` | Paginated indexed records by source family; these read graph data rather than calling upstream APIs. | -| `GET /api/integrations/status` | Connector configuration and recorded sync/index status; not an active provider probe. | +| `GET /api/integrations/status` | Bounded connector sample check plus recorded sync/index status. | +| `POST /api/integrations/:source/sync` | Administrative source sync; requires `RAG_ADMIN_API_KEY`. | | `GET /api/product-progress/graph` | Product Progress graph in the common node/edge response shape. | -| `POST /api/product-progress/tickets`, `/activities`, `/workflow` | Validated product/ticket/activity processing workflows. | +| `POST /api/product-progress/tickets`, `/activities`, `/workflow` | Validated graph-writing workflows; requires `RAG_ADMIN_API_KEY`. | | `POST /api/assistant/query` | OrgMind structured graph answer; accepts `{ question, conversationId?, context? }`. | | `GET /api/assistant/conversations[/:conversationId]` | Conversation listing or persisted turns for the supplied workspace identity. | | `POST /api/rag/query` | Company Brain query; accepts `{ question, filters?: { sourceTypes? } }`. Returns `{ answer, sources, retrieval, evidenceFound }`. | diff --git a/backend/src/agents/organization-intelligence/agent.ts b/backend/src/agents/organization-intelligence/agent.ts index 193a545..d900a71 100644 --- a/backend/src/agents/organization-intelligence/agent.ts +++ b/backend/src/agents/organization-intelligence/agent.ts @@ -6,6 +6,8 @@ import { GeminiEmbeddingService } from './embeddings/gemini-embeddings.js'; import { runIngestionGraph, type IngestionGraphConfig } from './workflow/ingestion-graph.js'; import { runRetrievalGraph, type RetrievalGraphConfig } from './workflow/retrieval-graph.js'; +const errorType = (error: unknown) => error instanceof Error && /^[A-Za-z][A-Za-z0-9]*Error$/.test(error.name) ? error.name : 'UnknownError'; + export interface Agent1Options { graph?: Graph; docsDir?: string; @@ -86,9 +88,9 @@ export class OrganizationIntelligenceAgent { console.log(`RAG embedding refresh complete: ${result.stats.totalChunks} chunks indexed.`); return runRetrievalGraph(input, config); } - console.error('RAG embedding refresh did not produce a searchable index:', result.errors); + console.error('RAG embedding refresh did not produce a searchable index:', { errorCount: result.errors.length }); } catch (error) { - console.error('RAG embedding refresh failed:', error); + console.error('RAG embedding refresh failed:', { errorType: errorType(error) }); } } } diff --git a/backend/src/agents/organization-intelligence/workflow/retrieval-graph.ts b/backend/src/agents/organization-intelligence/workflow/retrieval-graph.ts index f9848d0..c593af5 100644 --- a/backend/src/agents/organization-intelligence/workflow/retrieval-graph.ts +++ b/backend/src/agents/organization-intelligence/workflow/retrieval-graph.ts @@ -394,12 +394,13 @@ export function createRetrievalWorkflow(config: RetrievalGraphConfig = {}) { status: 'completed' as const }; } catch (error) { - console.warn('Model synthesis failed:', error instanceof Error ? error.message : String(error)); + const errorType = error instanceof Error && /^[A-Za-z][A-Za-z0-9]*Error$/.test(error.name) ? error.name : 'UnknownError'; + console.warn('Model synthesis failed:', { errorType }); return { answer: 'I found relevant company records, but Bedrock could not generate an answer. Please check the backend console and try again.', uncertainty: 'Check Bedrock model access and inference quotas for the configured AWS region.', status: 'partial' as const, - errors: [`Model synthesis error: ${error instanceof Error ? error.message : String(error)}`] + errors: ['Model synthesis failed. Check backend model configuration and service availability.'] }; } }) diff --git a/backend/src/api/app.ts b/backend/src/api/app.ts index 3815e57..4ea00f7 100644 --- a/backend/src/api/app.ts +++ b/backend/src/api/app.ts @@ -22,9 +22,12 @@ const filterSchema = z.object({ departmentId:idSchema.optional(), teamId:idSchem const sourceRecordsQuerySchema = z.object({ q:z.string().trim().max(200).optional(), limit:z.coerce.number().int().min(1).max(200).default(100), offset:z.coerce.number().int().min(0).max(100_000).default(0) }).strict(); const supported = new Set(['Company','Employee','Department','Team','Project','Repository','Service','Document','Decision','Incident','Task','PullRequest','Channel','Message','Product','ProductModule','Feature','FeatureRequest','Ticket','Bug','Release','Commit','Comment','Activity','Review']); const parse = (schema:z.ZodType, input:unknown): T => schema.parse(input); +const logFailure = (event: string, error: unknown) => console.error(event, { + errorType: error instanceof Error && /^[A-Za-z][A-Za-z0-9]*Error$/.test(error.name) ? error.name : 'UnknownError' +}); const sendError = (res:Response, error:unknown) => { if (error instanceof z.ZodError) return res.status(400).json({ error:'Invalid request', details:error.flatten() }); - console.error(error); return res.status(503).json({ error:'Database request failed' }); + logFailure('Database request failed', error); return res.status(503).json({ error:'Database request failed' }); }; const one = async (graph:Graph, label:string, id:string) => rows(await query(graph,`MATCH (n:${label} {id:$id}) RETURN n`,{id}))[0]?.n; type ApiGraphNode = { id:string; labels:string[]; properties:Record }; @@ -170,7 +173,7 @@ export function createApp(graph:Graph, options: { ingestion?: IngestionWorkflowD } return true; }; - app.get('/health',async (_req,res)=>{ try { await query(graph,'RETURN 1 AS connected'); res.json({status:'ok',database:'connected'}); } catch(error){ console.error(error); res.status(503).json({status:'degraded',database:'unavailable'}); } }); + app.get('/health',async (_req,res)=>{ try { await query(graph,'RETURN 1 AS connected'); res.json({status:'ok',database:'connected'}); } catch(error){ logFailure('Health check failed', error); res.status(503).json({status:'degraded',database:'unavailable'}); } }); const dashboardResponse = async () => { const [company, organization, productProgress] = await Promise.all([ one(graph,'Company','hirenova').catch(() => undefined), @@ -189,6 +192,7 @@ export function createApp(graph:Graph, options: { ingestion?: IngestionWorkflowD catch (error) { return sendError(res, error); } }); app.post('/api/integrations/:source/sync', async (req, res) => { + if (!adminOwner(req, res)) return; const params = z.object({ source: z.enum(mockSources) }).safeParse(req.params); const body = z.object({ limit: z.coerce.number().int().min(1).max(1000).default(500) }).strict().safeParse(req.body ?? {}); const parsed = params.success && body.success ? { success: true as const, data: { source: params.data.source, limit: body.data.limit } } : { success: false as const }; @@ -205,7 +209,9 @@ export function createApp(graph:Graph, options: { ingestion?: IngestionWorkflowD const result = await runIngestionWorkflow({ source, limit }, dependencies); const processed = result.outcomes.filter((item) => item.status === 'processed').length; const skipped = result.outcomes.filter((item) => item.status === 'skipped').length; - const failed = result.outcomes.filter((item) => item.status === 'failed' || item.status === 'unresolved').length; + const failedOutcomeIds = new Set(result.outcomes.filter((item) => item.status === 'failed' || item.status === 'unresolved').map((item) => item.sourceRecordId)); + const processedRecordIds = new Set(result.outcomes.filter((item) => item.status === 'processed').map((item) => item.sourceRecordId)); + const outcomeRecordIds = new Set(result.outcomes.map((item) => item.sourceRecordId)); const recordIds = [...new Set(result.documents.map((document) => document.sourceRecordId))]; const confirmedIds = options.integrations?.indexedSourceRecordIds ? await options.integrations.indexedSourceRecordIds(source, recordIds) @@ -213,15 +219,16 @@ export function createApp(graph:Graph, options: { ingestion?: IngestionWorkflowD ? recordIds.slice(0, await options.integrations.indexedSourceRecordCount(source, recordIds)) : await indexedSourceRecordIds(source, recordIds); const confirmedIdSet = new Set(confirmedIds); - const confirmedIndexed = confirmedIdSet.size; - const indexingConfirmed = confirmedIndexed >= recordIds.length; - const succeeded = (result.status === 'processed' || result.status === 'empty') && result.errors.length === 0 && failed === 0 && indexingConfirmed; + const indexedProcessedIds = [...processedRecordIds].filter((id) => confirmedIdSet.has(id)); + const persistedThisRun = indexedProcessedIds.length; + const indexingConfirmed = indexedProcessedIds.length === processedRecordIds.size; + const succeeded = (result.status === 'processed' || result.status === 'empty') && result.errors.length === 0 && failedOutcomeIds.size === 0 && indexingConfirmed; const previous = integrationRuns.get(source); - const failedOutcomeIds = new Set(result.outcomes.filter((item) => item.status === 'failed' || item.status === 'unresolved').map((item) => item.sourceRecordId)); - const missingIndexedIds = recordIds.filter((id) => !confirmedIdSet.has(id)); - const runFailed = new Set([...failedOutcomeIds, ...missingIndexedIds]).size; - const runState = succeeded ? 'completed' as const : confirmedIndexed > 0 ? 'partial' as const : 'failed' as const; - const run = { state: runState, fetched: result.documents.length, processed, persisted: confirmedIndexed, indexedThisRun: confirmedIndexed, skipped, failed: runFailed, attemptedAt: previous?.attemptedAt ?? new Date().toISOString(), lastSuccessfulSync: succeeded ? new Date().toISOString() : previous?.lastSuccessfulSync ?? null, ...(result.errors.length ? { error: result.errors.join(' ') } : !indexingConfirmed ? { error: `Indexed verification failed: only ${confirmedIndexed} of ${recordIds.length} fetched source record(s) were confirmed in the company graph.` } : runFailed ? { error: `${runFailed} unique source record(s) could not be fully processed or verified in the company graph.` } : {}) }; + const missingIndexedIds = [...processedRecordIds].filter((id) => !confirmedIdSet.has(id)); + const missingOutcomeIds = recordIds.filter((id) => !outcomeRecordIds.has(id)); + const runFailed = new Set([...failedOutcomeIds, ...missingIndexedIds, ...missingOutcomeIds]).size; + const runState = succeeded ? 'completed' as const : persistedThisRun > 0 ? 'partial' as const : 'failed' as const; + const run = { state: runState, fetched: result.documents.length, processed, persisted: persistedThisRun, indexedThisRun: persistedThisRun, skipped, failed: runFailed, attemptedAt: previous?.attemptedAt ?? new Date().toISOString(), lastSuccessfulSync: succeeded ? new Date().toISOString() : previous?.lastSuccessfulSync ?? null, ...(result.errors.length ? { error: result.errors.join(' ') } : !indexingConfirmed ? { error: `Indexed verification failed: only ${indexedProcessedIds.length} of ${processedRecordIds.size} successfully processed source record(s) were confirmed in the company graph.` } : runFailed ? { error: `${runFailed} unique source record(s) could not be fully processed or verified in the company graph.` } : {}) }; integrationRuns.set(source, run); await saveIntegrationRun(source, run); const indexed = await indexedCount(source); @@ -265,7 +272,7 @@ export function createApp(graph:Graph, options: { ingestion?: IngestionWorkflowD try { return res.json(await fetchEntireGraph(graph)); } catch (error) { - console.error('Unable to fetch complete graph:', error); + logFailure('Unable to fetch complete graph', error); return res.status(500).json({ error: 'Unable to fetch graph' }); } }); @@ -299,9 +306,10 @@ export function createApp(graph:Graph, options: { ingestion?: IngestionWorkflowD app.get('/api/search',async(req,res)=>{try{const q=parse(z.string().trim().min(1).max(100),req.query.q);const result=await query(graph,'MATCH (n) WHERE (n.name IS NOT NULL AND toLower(n.name) CONTAINS toLower($q)) OR (n.title IS NOT NULL AND toLower(n.title) CONTAINS toLower($q)) OR (n.email IS NOT NULL AND toLower(n.email) CONTAINS toLower($q)) RETURN labels(n)[0] AS entityType,n LIMIT 50',{q});return res.json(rows(result).map((item)=>({...item,n:item.n && 'properties' in item.n ? item.n.properties : item.n})));}catch(e){return sendError(res,e);}}); app.get('/api/product-progress/graph', async (_req, res) => { try { return res.json(await fetchProductProgressGraph(graph)); } - catch (error) { console.error('Unable to fetch Product Progress graph:', error); return res.status(503).json({ error: 'Product Progress graph unavailable' }); } + catch (error) { logFailure('Unable to fetch Product Progress graph', error); return res.status(503).json({ error: 'Product Progress graph unavailable' }); } }); const processWorkflow = (forcedType?: 'ticket' | 'feature-update') => async (req:Request, res:Response) => { + if (!adminOwner(req, res)) return; try { const body = parse(workflowBodySchema, { ...req.body, ...(forcedType ? { type: forcedType } : {}) }); const result = await runOrgInGraphsWorkflow(body, { graph }); @@ -375,7 +383,7 @@ export function createApp(graph:Graph, options: { ingestion?: IngestionWorkflowD return res.status(result.status === 'failed' ? 503 : 200).json(response); } catch (error) { if (error instanceof z.ZodError) return res.status(400).json({ error: 'Invalid request', details: error.flatten() }); - console.error('Unable to process ingestion request:', error); + logFailure('Unable to process ingestion request', error); return res.status(500).json({ error: 'Ingestion processing failed' }); } }); @@ -390,7 +398,7 @@ export function createApp(graph:Graph, options: { ingestion?: IngestionWorkflowD const result = await agent1.ingest({ forceReindex: parsed.data.force, selectedSources: parsed.data.sources }); return res.status(result.status === 'failed' ? 503 : 200).json(result); } catch (error) { - console.error('Agent 1 ingestion failed:', error); + logFailure('Agent 1 ingestion failed', error); return res.status(503).json({ status: 'failed', errors: ['Ingestion workflow execution failed safely.'] }); } finally { ragWriteActive = false; } }; @@ -404,7 +412,7 @@ export function createApp(graph:Graph, options: { ingestion?: IngestionWorkflowD const sync = options.ragSyncWorker ? await options.ragSyncWorker.status() : { enabled: false, running: false, pending: 0, processing: 0, completed: 0, failed: 0, retries: 0, lastObservedChange: null, lastSuccessfulSync: null, lastError: null, workerLagMs: 0 }; return res.json({ status: 'ready', stats, indexedDocuments: stats.totalSources, indexedChunks: stats.totalChunks, pendingWork: (ragWriteActive ? 1 : 0) + sync.pending + sync.processing, failedWork: stats.failed, lastSuccessfulIngestion: stats.lastIngestedAt ?? null, sync, agent: 'organization-intelligence', embeddingModel: agent1.embeddingService.model, vectorStore: agent1.vectorStore instanceof Object && 'healthy' in agent1.vectorStore ? (agent1.vectorStore.healthy ? 'healthy' : 'unavailable') : 'local' }); } catch (error) { - console.error('Agent 1 status failed:', error); + logFailure('Agent 1 status failed', error); return res.status(500).json({ error: 'Failed to retrieve indexing status.' }); } }; @@ -426,22 +434,22 @@ export function createApp(graph:Graph, options: { ingestion?: IngestionWorkflowD if (result.status === 'failed' || result.stats.totalChunks === 0) { console.error('Initial Organization RAG indexing did not produce searchable chunks:', { status: result.status, - stats: result.stats, - errors: result.errors + failedSources: result.stats.failed, + errorCount: result.errors.length }); } else { console.log(`Initial Organization RAG index ready: ${result.stats.totalChunks} chunks from ${result.stats.totalSources} sources.`); } }).catch((error) => { - console.error('Initial Organization RAG indexing failed:', error); + logFailure('Initial Organization RAG indexing failed', error); }).finally(() => { initialRagIndex = undefined; }); await initialRagIndex; } const response = await agent1.query({ ...parsed.data, orgId: process.env.ORGANIZATION_ID ?? 'hirenova' }); - if (response.status !== 'completed') console.error('Organization RAG query returned a degraded result:', { query: parsed.data.query, status: response.status, errors: response.errors, uncertainty: response.uncertainty }); + if (response.status !== 'completed') console.error('Organization RAG query returned a degraded result:', { status: response.status, errorCount: response.errors?.length ?? 0 }); return res.status(response.status === 'failed' ? 503 : 200).json(response); } catch (error) { - console.error('Agent 1 query failed:', { query: parsed.data.query, error }); + logFailure('Agent 1 query failed', error); return res.status(503).json({ answer: 'The organization intelligence query could not be completed safely.', sources: [], @@ -460,7 +468,7 @@ export function createApp(graph:Graph, options: { ingestion?: IngestionWorkflowD const sources = await agent1.getSources(); return res.json({ sources, count: sources.length }); } catch (error) { - console.error('Agent 1 getSources failed:', error); + logFailure('Agent 1 getSources failed', error); return res.status(500).json({ error: 'Failed to retrieve indexed sources.' }); } }; @@ -475,7 +483,7 @@ export function createApp(graph:Graph, options: { ingestion?: IngestionWorkflowD const result = await agent1.reindex(sourcePath); return res.status(result.status === 'failed' ? 503 : 200).json(result); } catch (error) { - console.error('Agent 1 reindexing failed:', error); + logFailure('Agent 1 reindexing failed', error); return res.status(503).json({ status: 'failed', errors: ['Reindexing failed safely.'] }); } finally { ragWriteActive = false; } }; diff --git a/backend/tests/api.test.ts b/backend/tests/api.test.ts index c818a02..128caa7 100644 --- a/backend/tests/api.test.ts +++ b/backend/tests/api.test.ts @@ -128,9 +128,9 @@ test("Product Progress endpoints validate payloads and keep graph data scoped", assert.deepEqual(graphPayload.nodes.map((node) => node.id), ['product-orbit', 'emp-01']); assert.deepEqual(graphPayload.edges[0], { id: 'OWNED_BY:product-orbit:emp-01', type: 'OWNED_BY', source: 'product-orbit', target: 'emp-01', properties: { graphDomain: 'product-progress' } }); - const invalid = await fetch(`http://127.0.0.1:${address.port}/api/product-progress/tickets`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text: '' }) }); + const invalid = await fetch(`http://127.0.0.1:${address.port}/api/product-progress/tickets`, { method: 'POST', headers: ingestionAdminHeaders, body: JSON.stringify({ text: '' }) }); assert.equal(invalid.status, 400); - const processed = await fetch(`http://127.0.0.1:${address.port}/api/product-progress/activities`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text: 'PR #142 was merged.' }) }); + const processed = await fetch(`http://127.0.0.1:${address.port}/api/product-progress/activities`, { method: 'POST', headers: ingestionAdminHeaders, body: JSON.stringify({ text: 'PR #142 was merged.' }) }); assert.equal(processed.status, 200); const payload = await processed.json() as { eventType: string; unresolvedReferences: Array<{ text: string }> }; assert.equal(payload.eventType, 'feature-update'); @@ -240,6 +240,11 @@ test("provider synchronization rejects missing identity or admin credentials", a assert.equal(noIdentity.status, 401); const noAdminKey = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json', 'x-orgmind-user-id': 'demo@example.com' }, body: JSON.stringify({ source: 'jira' }) }); assert.equal(noAdminKey.status, 403); + const base = url.replace('/api/ingestion/process', ''); + const publicSync = await fetch(`${base}/api/integrations/jira/sync`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ limit: 1 }) }); + assert.equal(publicSync.status, 401); + const missingSyncAdmin = await fetch(`${base}/api/integrations/jira/sync`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'x-orgmind-user-id': 'demo@example.com' }, body: JSON.stringify({ limit: 1 }) }); + assert.equal(missingSyncAdmin.status, 403); assert.equal(fetched, false); } finally { await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); } }); @@ -348,7 +353,7 @@ test("integration sync only reports success after fetched records are confirmed const successAddress = successServer.address(); assert.ok(successAddress && typeof successAddress !== 'string'); try { - const response = await fetch(`http://127.0.0.1:${successAddress.port}/api/integrations/jira/sync`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ limit: 1 }) }); + const response = await fetch(`http://127.0.0.1:${successAddress.port}/api/integrations/jira/sync`, { method: 'POST', headers: ingestionAdminHeaders, body: JSON.stringify({ limit: 1 }) }); assert.equal(response.status, 200, JSON.stringify(await response.clone().json())); const payload = await response.json() as { syncState: string; fetched: number; processed: number; indexed: number; failed: number; lastSuccessfulSync: string | null }; assert.deepEqual([payload.syncState, payload.fetched, payload.processed, payload.indexed, payload.failed], ['completed', 1, 1, 1, 0]); @@ -360,7 +365,7 @@ test("integration sync only reports success after fetched records are confirmed const failureAddress = failureServer.address(); assert.ok(failureAddress && typeof failureAddress !== 'string'); try { - const response = await fetch(`http://127.0.0.1:${failureAddress.port}/api/integrations/jira/sync`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ limit: 1 }) }); + const response = await fetch(`http://127.0.0.1:${failureAddress.port}/api/integrations/jira/sync`, { method: 'POST', headers: ingestionAdminHeaders, body: JSON.stringify({ limit: 1 }) }); assert.equal(response.status, 502); const payload = await response.json() as { syncState: string; failed: number; lastSuccessfulSync: string | null; error: string }; assert.equal(payload.syncState, 'failed'); @@ -390,13 +395,13 @@ test("integration status separates reachable connectivity from a partial ingesti const address = server.address(); assert.ok(address && typeof address !== 'string'); try { - const response = await fetch(`http://127.0.0.1:${address.port}/api/integrations/jira/sync`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ limit: 2 }) }); + const response = await fetch(`http://127.0.0.1:${address.port}/api/integrations/jira/sync`, { method: 'POST', headers: ingestionAdminHeaders, body: JSON.stringify({ limit: 2 }) }); assert.equal(response.status, 207); const payload = await response.json() as { syncState: string; fetched: number; processed: number; persisted: number; indexedThisRun: number; indexed: number; failed: number }; - assert.deepEqual([payload.syncState, payload.fetched, payload.processed, payload.persisted, payload.indexedThisRun, payload.indexed, payload.failed], ['partial', 2, 1, 2, 2, 9, 1]); + assert.deepEqual([payload.syncState, payload.fetched, payload.processed, payload.persisted, payload.indexedThisRun, payload.indexed, payload.failed], ['partial', 2, 1, 1, 1, 9, 1]); const status = await fetch(`http://127.0.0.1:${address.port}/api/integrations/status`); const source = (await status.json() as { sources: Array<{ source: string; connectionState: string; syncState: string; indexed: number; indexedThisRun: number }> }).sources.find((entry) => entry.source === 'jira'); - assert.deepEqual([source?.connectionState, source?.syncState, source?.indexed, source?.indexedThisRun], ['Connected', 'partial', 9, 2]); + assert.deepEqual([source?.connectionState, source?.syncState, source?.indexed, source?.indexedThisRun], ['Connected', 'partial', 9, 1]); } finally { await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); } }); @@ -423,7 +428,7 @@ test("integration status remains available after app restart via graph-backed la const firstAddress = first.address(); assert.ok(firstAddress && typeof firstAddress !== 'string'); try { - const response = await fetch('http://127.0.0.1:' + firstAddress.port + '/api/integrations/jira/sync', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ limit: 1 }) }); + const response = await fetch('http://127.0.0.1:' + firstAddress.port + '/api/integrations/jira/sync', { method: 'POST', headers: ingestionAdminHeaders, body: JSON.stringify({ limit: 1 }) }); assert.equal(response.status, 200); } finally { await new Promise((resolve, reject) => first.close((error) => error ? reject(error) : resolve())); } diff --git a/docs/api-capability-inventory.md b/docs/api-capability-inventory.md index a8184c5..f99961d 100644 --- a/docs/api-capability-inventory.md +++ b/docs/api-capability-inventory.md @@ -1,6 +1,6 @@ # Backend API capability inventory -The backend exposes no OpenAPI document. This inventory is based on the current Express routes, workflow schemas, agent implementations, and existing response types. The backend has no authentication middleware; the local demo login is not an access boundary. +The backend exposes no OpenAPI document. This inventory is based on the current Express routes, workflow schemas, agent implementations, and response types. There is no general user authentication or tenant isolation: the local demo login is not an access boundary, and clients can spoof `x-orgmind-user-id`. Graph and assistant data must not be treated as private until identities are verified server-side. Graph-writing and administrative routes require `RAG_ADMIN_API_KEY` in addition to the identity header. | Endpoint | Contract / behavior | Frontend use | |---|---|---| @@ -13,7 +13,8 @@ The backend exposes no OpenAPI document. This inventory is based on the current | `GET /api/tickets?q=&limit=&offset=` | Paginated Jira `Ticket` and `Bug` graph records | Tickets page | | `GET /api/conversations?q=&limit=&offset=` | Paginated Slack `Message` records and linked channels | Conversations page | | `GET /api/proactive-insights?q=&limit=&offset=` | Paginated GitHub pull request, issue, and commit records | Proactive Insights source view (not a generated risk assessment) | -| `GET /api/integrations/status` | Connector configuration, verified indexed counts, last sync outcome, and ten recent sanitized sync runs; it does not actively probe the upstream provider | Integrations and Sync & Data pages | +| `GET /api/integrations/status` | Bounded fetch of up to one sample record per source, configuration, cumulative verified indexed counts, latest run outcome, and ten recent sanitized sync runs. Sample reachability and ingestion health are separate. | Integrations and Sync & Data pages | +| `POST /api/integrations/:source/sync` | Validated sync for one source; requires `RAG_ADMIN_API_KEY`. The demo browser cannot start syncs. | Backend operator only | | `GET /api/graph/neighborhood/:entityType/:entityId` | Direct neighbors and relationship direction, capped at 100 | Graph expansion; entity, ownership, decision, and impact views | | `GET /api/search?q=` | Up to 50 graph records matching name, title, or email | Global search and assistant citations | | `GET /api/employees[?departmentId&teamId&employmentStatus]` | Employee list/filter | People directory | @@ -21,9 +22,9 @@ The backend exposes no OpenAPI document. This inventory is based on the current | `GET /api/departments[/:departmentId]` | Department list and department/team/employee details | Department directory/details | | `GET /api/projects[/:projectId]` | Project list/details with owners, people, tasks, PRs, and services | Project directory/details and assistant cards | | `GET /api/product-progress/graph` | Product Progress graph in the standard graph response shape | Product Progress graph | -| `POST /api/product-progress/tickets` | Validated ticket identification and graph persistence | Existing workflow capability; no automatic assistant write action | -| `POST /api/product-progress/activities` | Validated activity processing and graph persistence | Existing workflow capability; no automatic assistant write action | -| `POST /api/product-progress/workflow` | Routes ticket or feature update input | Existing workflow capability | +| `POST /api/product-progress/tickets` | Validated ticket identification and graph persistence; requires `RAG_ADMIN_API_KEY` | Existing workflow capability; no automatic assistant write action | +| `POST /api/product-progress/activities` | Validated activity processing and graph persistence; requires `RAG_ADMIN_API_KEY` | Existing workflow capability; no automatic assistant write action | +| `POST /api/product-progress/workflow` | Routes ticket or feature update input; requires `RAG_ADMIN_API_KEY` | Existing workflow capability | | `POST /api/assistant/query` | LangGraph retrieval and optional synthesis; validated structured answer with graph evidence and execution stages | Right-side assistant drawer and safe response renderer | `GET /api/assistant/conversations` | Lists up to 50 conversations for the supplied workspace identity | Assistant conversation switcher | `GET /api/assistant/conversations/:conversationId` | Returns persisted turns after checking conversation ownership | Restore a prior assistant conversation @@ -33,9 +34,11 @@ The backend exposes no OpenAPI document. This inventory is based on the current The Documents page reads `GET /api/documents`; the Tickets page reads `GET /api/tickets`. Both routes paginate and filter persisted source records, and neither calls upstream services. Conversations reads `GET /api/conversations`, and Proactive Insights reads `GET /api/proactive-insights`. The overview page uses `GET /api/dashboard` for the company, organization and product graphs, counts, and dashboard cards. Backend read failures are displayed in the relevant page. Browser-triggered sync is disabled because the frontend has no trusted operator authentication channel; sync requires a backend-side caller holding `RAG_ADMIN_API_KEY`. +For an integration sync response, `fetched` counts records returned by the connector, `processed` counts successful processing outcomes, and `persisted`/`indexedThisRun` count processed source IDs that the post-run graph check confirms. `indexed` is the cumulative distinct verified source-record count. Failed and skipped records are reported separately. A prior graph record that is merely present does not count as persisted by a failed current processing outcome. + Confluence fetching uses the configured server-side Confluence REST API connection (`CONFLUENCE_BASE_URL`, `CONFLUENCE_EMAIL`, and `CONFLUENCE_API_TOKEN` or `CONFLUENCE_API_KEY`) when `MOCK_API_BASE_URL` is not set. Jira fetching uses the configured connector at `MOCK_API_BASE_URL/api/jira/issues` with `JIRA_API_KEY` when supplied; a direct Jira Cloud REST client is not implemented. The route catalog's `/api/confluence/*` and `/api/jira/*` entries describe connector/source capabilities and are not Express routes exposed by this backend. The supplied Jira response contains `rawSeedId`, which marks the current records as fixtures; the ingestion workflow rejects these records and the Tickets API intentionally does not show them. Configure a connector that returns genuine Jira records marked with verified provenance before expecting them in Tickets. -The same ingestion flow now persists Slack messages as graph `Message` nodes with channel relationships and GitHub pull requests, issues, and commits as graph source records. The frontend maps Conversations to Slack `Message` records and Proactive Insights to GitHub records. The latter is a source-backed activity view; the backend does not currently calculate or persist risk insights from GitHub data. Source refreshes use the route catalog contract (`/api/slack/conversations.list` and `/api/slack/conversations.history`; `/api/github/pulls`, `/api/github/issues`, and `/api/github/commits`) through `MOCK_API_BASE_URL`, which defaults to `http://localhost:8787` for these connector APIs. +The same ingestion flow persists Slack messages as graph `Message` nodes with channel relationships and GitHub pull requests, issues, and commits as graph source records. The frontend maps Conversations to Slack `Message` records and Proactive Insights to GitHub records. The latter is a source-backed activity view; the backend does not currently calculate or persist risk insights from GitHub data. Source refreshes use the route catalog contract (`/api/slack/conversations.list` and `/api/slack/conversations.history`; `/api/github/pulls`, `/api/github/issues`, and `/api/github/commits`) through the explicitly configured `MOCK_API_BASE_URL`; there is no implicit connector URL fallback. Per-source credentials are backend-only: `CONFLUENCE_API_KEY` (or the existing `CONFLUENCE_API_TOKEN` for direct Atlassian access), `JIRA_API_KEY`, `SLACK_API_KEY`, and `GITHUB_API_KEY`. For the route-catalog connector, a non-URL source key is sent in `x-api-key` and Bearer authorization headers, but it does not establish record provenance. Generic route-catalog responses are classified as synthetic even if the response asserts otherwise. Only the direct Confluence REST client marks records verified, and it excludes explicit seed markers. Graph reads are served from FalkorDB, so page navigation does not refetch upstream APIs. @@ -59,7 +62,7 @@ These response shapes are useful connector contracts. The stats and source cards The ingestion endpoint accepts `confluence`, `jira`, `github`, or `slack`, with an optional limit from 1 to 100. Confluence uses `CONFLUENCE_BASE_URL`, `CONFLUENCE_EMAIL`, and `CONFLUENCE_API_TOKEN` on the backend when no mock source URL is configured; pages are stored as verified `Document` nodes. Jira records route through ticket identification. Some GitHub and Slack records route through feature-update processing. Other sources use the backend `MOCK_API_BASE_URL`. Credentials are not accepted in browser requests. The endpoint returns counts and outcomes, not source content. Authorized runs are summarized in `IntegrationSyncRun` graph nodes and exposed through the integration status response. Synthetic records are counted as fetched, rejected before processing, and never written. -In the local backend environment inspected on 2026-10-18, Confluence has no configured credentials or base URL. Jira, GitHub, and Slack use URL-valued `*_API_KEY` entries that point to the same route-catalog tunnel, not provider API tokens. The tunnel has been probed and returns seed-marked fixture data. Local authorized sync attempts reject those records and preserve zero verified writes. Native Jira, GitHub, and Slack provider REST clients are not part of this backend; setting a secret token alone does not turn the route-catalog connector into an upstream integration. Configure a trusted live connector with provider credentials and verifiable upstream records before expecting nonzero live source counts. +Local runtime status queried on 2026-10-18 reported Confluence as a live provider with a completed sync (11 fetched, processed, persisted, and indexed; 0 failed), and Jira, GitHub, and Slack as mock sources. Their sample checks were reachable, while their latest syncs were partial: Jira 129 fetched / 109 processed / 109 persisted-reported / 20 failed / 109 cumulative indexed; GitHub 444 / 405 / 211 / 32 / 211; Slack 217 / 112 / 217 / 105 / 217. The stored Jira/GitHub/Slack persisted values predate the corrected run aggregation and may include IDs already indexed before those attempts; they must not be interpreted as current-run success totals. No new sync was run during this audit, so corrected persisted-this-run values remain unknown until the next authorized sync. Jira, GitHub, and Slack use the seeded mock API, not native provider connections. These counts are a point-in-time observation; the persistent latest-run record changes only after an authorized sync. The assistant query endpoint `POST /api/assistant/query` runs a LangGraph workflow: validate, classify, retrieve graph entities and direct neighbors, optionally synthesize with the configured model, and validate a structured response. With model configuration disabled or unavailable, it returns deterministic summaries, verified graph citations, uncertainty, and execution stage statuses. It supports company overview, entity lookup, ownership lookup, decision investigation, impact traversal, graph query, and general explanation. Impact output explicitly reports direct graph traversal and does not claim a simulation. The response renderer uses an allowlist of predefined React views, and citation links resolve to graph-backed entities or their indexed source URL. diff --git a/frontend/README.md b/frontend/README.md index 29fb9dd..ce33c23 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -29,7 +29,7 @@ npm start ## Configuration -Set `NEXT_PUBLIC_API_BASE_URL` in `.env.local` to the browser-accessible backend URL. The default is `http://localhost:8080`. +Set `NEXT_PUBLIC_API_BASE_URL` in `.env.local` to the browser-accessible backend URL. Development falls back to `http://localhost:8080`; production builds fail requests with a clear configuration error when the variable is missing instead of calling the visitor's localhost. The backend CORS allowlist includes `https://orgingraphs-ysq7.vercel.app`. Set `FRONTEND_ORIGINS` in the backend environment to a comma-separated list for local development or any additional deployment origins. @@ -39,6 +39,6 @@ The backend CORS allowlist includes `https://orgingraphs-ysq7.vercel.app`. Set ` - The dashboard and directories use records returned by the backend. Initial graph layout is deterministic and limited to the first 24 employees; select and expand nodes to progressively load real neighborhoods. - Search uses `GET /api/search` and supports Ctrl+K / Cmd+K. Result types without a dedicated detail endpoint open in the graph. - The assistant builds allowlisted entity, ownership, decision evidence, direct graph impact, and graph query cards from search, entity detail, and neighborhood APIs. Citations link to indexed graph records. This backend does not expose a conversational answer endpoint; the UI does not claim that a LangGraph ingestion run is an assistant answer or an impact simulation. -- Documents, Tickets, Conversations, and Proactive Insights display only verified records read from the backend graph. `/api/integrations/status` reports verified record counts and the last ten authorized sync outcomes; it does not actively probe provider connectivity. Sync requires a backend operator with `RAG_ADMIN_API_KEY` and is not exposed to the demo browser session. The configured Jira connector returns fixture `rawSeedId` values, so those issues are deliberately excluded from Tickets until a real verified Jira connector is configured. +- Documents, Tickets, Conversations, and Proactive Insights display only verified records read from the backend graph. `/api/integrations/status` reports verified record counts and the last ten recorded sync outcomes; connectivity checks are separate. Sync requires a backend operator with `RAG_ADMIN_API_KEY` and is not exposed to the demo browser session. The configured Jira connector returns fixture `rawSeedId` values, so those issues are deliberately excluded from Tickets until a real verified Jira connector is configured. - The assistant drawer calls `POST /api/assistant/query` and renders validated LangGraph response types with source citations, related entities, direct graph relationships, uncertainty, and execution stages. When the default model is disabled or unavailable, the backend provides a deterministic evidence-based response. - Neighborhood relationships are labeled with their actual edge direction. Other relationships are displayed only when returned by the API or derivable from its list/detail fields. diff --git a/frontend/components/assistant/AgentResponseRenderer.tsx b/frontend/components/assistant/AgentResponseRenderer.tsx index 04a3e6b..b14776c 100644 --- a/frontend/components/assistant/AgentResponseRenderer.tsx +++ b/frontend/components/assistant/AgentResponseRenderer.tsx @@ -485,25 +485,6 @@ function isRagResponse(payload: unknown): payload is OrganizationRagResponse | R ); } -function TypedRagAnswer({ text, citationCount, onCitationClick }: { text: string; citationCount: number; onCitationClick: (number: number) => void }) { - const [visibleText, setVisibleText] = useState(""); - useEffect(() => { - if (window.matchMedia("(prefers-reduced-motion: reduce)").matches) { - setVisibleText(text); - return; - } - const characters = Array.from(text); - let visible = 0; - const timer = window.setInterval(() => { - visible = Math.min(characters.length, visible + 3); - setVisibleText(characters.slice(0, visible).join("")); - if (visible >= characters.length) window.clearInterval(timer); - }, 14); - return () => window.clearInterval(timer); - }, [text]); - return <>{visibleText.length < text.length &&