diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c35d10..bf68998 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,44 @@ here. The release version is defined in the workspace root `package.json`. ## [Unreleased] +## [0.4.1] - 2026-09-14 + +### Fixed + +- Anam, Avatar and Recall integrations appear in the integrations list in + every environment. The three types were marked private and only listed in + development and staging, so an integration created through a blueprint was + reachable by URL but missing from the list and its total. +- Hydration no longer fails on pages that link to the page's own origin or to + a portal or space host. The router read the request host cookie on the + server but not on the first client render, and the apex hooks were seeded + from server-only environment, so the server HTML never matched the browser. + Both now resolve after hydration, as `useHost` already did. +- Sitemap integrations accept source URLs up to 768 bytes. The `url` column + was a plain db string, so a long sitemap URL failed the onboarding wizard + with a 191-byte limit error. +- A stored image that fails to decode gets the generated icon thumbnail + without being reported as an error, and a GitHub integration reply that + fails on an expected GitHub 4xx is logged without being reported either. +- The Notion integration token column is optional, as the credential columns + of every other integration are, so creating one without a token no longer + fails in the database. A sync on an integration without a token answers + 409 instead of launching the crawler. Creating or updating a dataset record + with empty or blank text answers 400 instead of failing in the vector store. +- Creating a conversation message of type `activity`, or updating a message + so that it becomes one without its activity meta, answers 400 as the + stateless completion already did. A stored activity message without that + meta broke every later completion of that conversation. +- Code blocks highlight with shiki's JavaScript regex engine instead of the + WebAssembly one, so pages with code no longer fail in iOS Lockdown Mode. +- App actions report only unexpected failures. An expected answer such as + reaching the account limits is returned to the app without being reported + as an error. +- Integration setup, session create, record, conversation create and the + conversation completion routes report only unexpected failures. An answer + with a known code - not found, bad request, limits reached, a completion + timeout - reaches the caller as before without being reported as an error. + ## [0.4.0] - 2026-09-10 ### Changed diff --git a/package.json b/package.json index 5675b3b..9e137ea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "platform", - "version": "0.4.0", + "version": "0.4.1", "private": true, "license": "Apache-2.0", "packageManager": "pnpm@11.24.0", diff --git a/packages/db-spec/prisma/schema.prisma b/packages/db-spec/prisma/schema.prisma index 5d38f67..c24c44b 100644 --- a/packages/db-spec/prisma/schema.prisma +++ b/packages/db-spec/prisma/schema.prisma @@ -2202,7 +2202,7 @@ model SitemapIntegration { name String @default("") description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text - url String? + url String? @db.VarChar(768) glob String? @@ -2256,7 +2256,7 @@ model NotionIntegration { name String @default("") description String @default(dbgenerated("(_utf8mb4\\'\\')")) @db.Text - token String @db.Text /// @encrypted + token String? @db.Text /// @encrypted expiresIn Float? diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 7f1452a..5fe57e6 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -2256,7 +2256,7 @@ model NotionIntegration { name String @default("") description String @default("") - token String /// @encrypted + token String? /// @encrypted expiresIn Float? diff --git a/packages/db/src/constraints.ts b/packages/db/src/constraints.ts index 522a676..4a29f18 100644 --- a/packages/db/src/constraints.ts +++ b/packages/db/src/constraints.ts @@ -35,6 +35,12 @@ const SQLITE_MAX_LENGTH = 1000000000 */ export const MAX_DB_STRING_BYTES_LENGTH = SQLITE_MAX_LENGTH +/** + * The max length of what the blueprint declares `@db.VarChar(768)` - a source + * URL - stored here as an unconstrained TEXT column. + */ +export const MAX_DB_SOURCE_URL_BYTES_LENGTH = SQLITE_MAX_LENGTH + /** * The max length of what the blueprint declares `@db.Text`, stored here as an * unconstrained TEXT column. diff --git a/platform/components/IntegrationList.jsx b/platform/components/IntegrationList.jsx index a080e5c..e7a7b8b 100644 --- a/platform/components/IntegrationList.jsx +++ b/platform/components/IntegrationList.jsx @@ -17,8 +17,8 @@ import clsx from 'clsx' // Integrations render as a single merged list across every integration type. // The table below is the source of truth: it drives the combined GraphQL -// query, the type tag on every row, and the private (development and staging -// only) gating. +// query, the type tag on every row, and the optional `private` (development +// and staging only) gating; no type is gated at the moment. export const INTEGRATION_TYPES = [ { type: 'widget', connection: 'widgetIntegrations' }, { type: 'slack', connection: 'slackIntegrations' }, @@ -59,9 +59,9 @@ export const INTEGRATION_TYPES = [ }, { type: 'mcpserver', connection: 'mcpserverIntegrations' }, { type: 'skillserver', connection: 'skillserverIntegrations' }, - { type: 'anam', connection: 'anamIntegrations', private: true }, - { type: 'avatar', connection: 'avatarIntegrations', private: true }, - { type: 'recall', connection: 'recallIntegrations', private: true }, + { type: 'anam', connection: 'anamIntegrations' }, + { type: 'avatar', connection: 'avatarIntegrations' }, + { type: 'recall', connection: 'recallIntegrations' }, ] // @note every connection is capped at its first 100 integrations. When a diff --git a/platform/components/IntegrationList.utest.jsx b/platform/components/IntegrationList.utest.jsx index 42aaf6c..ccf532a 100644 --- a/platform/components/IntegrationList.utest.jsx +++ b/platform/components/IntegrationList.utest.jsx @@ -1,4 +1,4 @@ -import IntegrationList from './IntegrationList' +import IntegrationList, { INTEGRATION_TYPES } from './IntegrationList' import '@testing-library/jest-dom' import { fireEvent, render, screen, waitFor } from '@testing-library/react' @@ -340,22 +340,45 @@ describe('IntegrationList', () => { expect(screen.queryByText('Load more')).not.toBeInTheDocument() }) - it('should query the private connections only when they are shown', async () => { + it('should query every integration connection without the private flag', async () => { respondWith({}) - const { unmount } = render() + render() await waitFor(() => expect(mockFetch).toHaveBeenCalled()) - expect(lastQuery()).not.toContain('anamIntegrations') + for (const { connection } of INTEGRATION_TYPES) { + expect(lastQuery()).toContain(connection) + } + }) - unmount() + it('should query the private connections only when they are shown', async () => { + respondWith({}) + + const privateTypes = [ + ...INTEGRATION_TYPES, + { type: 'hidden', connection: 'hiddenIntegrations', private: true }, + ] + + INTEGRATION_TYPES.splice(0, INTEGRATION_TYPES.length, ...privateTypes) + + try { + const { unmount } = render() + + await waitFor(() => expect(mockFetch).toHaveBeenCalled()) + + expect(lastQuery()).not.toContain('hiddenIntegrations') + + unmount() - render() + render() - await waitFor(() => expect(mockFetch).toHaveBeenCalledTimes(2)) + await waitFor(() => expect(mockFetch).toHaveBeenCalledTimes(2)) - expect(lastQuery()).toContain('anamIntegrations') + expect(lastQuery()).toContain('hiddenIntegrations') + } finally { + INTEGRATION_TYPES.pop() + } }) }) diff --git a/platform/hooks/useHost.tsx b/platform/hooks/useHost.tsx index eae37c7..e2346cb 100644 --- a/platform/hooks/useHost.tsx +++ b/platform/hooks/useHost.tsx @@ -22,8 +22,8 @@ import { import { parse } from '@/lib/cookie' import { isProduction } from '@/lib/env' import { getExternalAPIHost } from '@/lib/host' -import { isLocalhost } from '@/lib/localhost' import { hostToHostname, normalizeRequestHost } from '@/lib/host.parse' +import { isLocalhost } from '@/lib/localhost' import useCookie from '@/hooks/useCookie' import useHydrated from '@/hooks/useHydrated' @@ -97,7 +97,11 @@ export function useAPIHost(): string { } export function usePortalApex(): string { - const [apex, setApex] = useState(portalApex || '') + // @note the apexes are server-only environment, so the constant is set on + // the server and empty in the browser - seed empty on both sides and resolve + // in the layout effect, or the server HTML never matches the first render + + const [apex, setApex] = useState('') useHydrationSafeLayoutEffect(() => { setApex(document.documentElement.dataset.portalApex || portalApex || '') @@ -107,7 +111,11 @@ export function usePortalApex(): string { } export function useSpaceApex(): string { - const [apex, setApex] = useState(spaceApex || '') + // @note the apexes are server-only environment, so the constant is set on + // the server and empty in the browser - seed empty on both sides and resolve + // in the layout effect, or the server HTML never matches the first render + + const [apex, setApex] = useState('') useHydrationSafeLayoutEffect(() => { setApex(document.documentElement.dataset.spaceApex || spaceApex || '') diff --git a/platform/hooks/useHost.utest.js b/platform/hooks/useHost.utest.js index 4f7ee76..feb31ec 100644 --- a/platform/hooks/useHost.utest.js +++ b/platform/hooks/useHost.utest.js @@ -446,6 +446,25 @@ describe('configured apexes', () => { expect(result.current).toBe('portal.example.com') }) + it('should seed the apexes empty on the server render', () => { + // @note the apexes are server-only environment - the browser bundle has + // none, so a seeded server render would never match the first client one + let portal = 'unset' + let space = 'unset' + + function Probe() { + portal = usePortalApex() + space = useSpaceApex() + + return null + } + + renderToString() + + expect(portal).toBe('') + expect(space).toBe('') + }) + it('should resolve the space apex from the document', () => { document.documentElement.dataset.spaceApex = 'space.brand.example' diff --git a/platform/hooks/useRouter.jsx b/platform/hooks/useRouter.jsx index 7066243..98baea0 100644 --- a/platform/hooks/useRouter.jsx +++ b/platform/hooks/useRouter.jsx @@ -28,6 +28,7 @@ import { useCookieHost, useSiteHost, } from '@/hooks/useHost' +import useHydrated from '@/hooks/useHydrated' import i18n from '@/i18n.config' import base from '@/next.config.d/base.config' @@ -169,10 +170,19 @@ export default function useRouter() { // host cookie carry them; the app tables hold hosts too and are looked up // by hostname - const cookieHost = useCookieHost() + const requestCookieHost = useCookieHost() const audienceHost = useAudienceHost() const siteHostRuntime = useSiteHost() + // @note the cookie is read from the request on the server but is not + // readable during the first client render, so it is ignored until hydration + // - otherwise the server strips own-origin hrefs the client keeps absolute + // and hydration fails - see useHost + + const hydrated = useHydrated() + + const cookieHost = hydrated ? requestCookieHost : '' + const host = cookieHost || audienceHost const hostname = hostToHostname(host) diff --git a/platform/hooks/useRouter.utest.jsx b/platform/hooks/useRouter.utest.jsx index 738db0b..ae1a45f 100644 --- a/platform/hooks/useRouter.utest.jsx +++ b/platform/hooks/useRouter.utest.jsx @@ -34,6 +34,8 @@ jest.mock('@/hooks/useHost', () => ({ useSiteHost: jest.fn(), })) +jest.mock('@/hooks/useHydrated', () => jest.fn(() => true)) + jest.mock('@/i18n.config', () => ({ __esModule: true, default: { locales: ['en'], defaultLocale: 'en', domainLocales: [] }, @@ -85,6 +87,22 @@ describe('useRouter href resolution by host', () => { jest.clearAllMocks() }) + describe('before hydration', () => { + it('ignores the request cookie host so the server and first client render agree', () => { + const useHydrated = require('@/hooks/useHydrated') + + useHydrated.mockReturnValueOnce(false) + + const router = setup({ cookieHostname: 'site.example.com' }) + + // @note the server reads the cookie from the request and the browser + // cannot until it has hydrated - the absolute href is left as is on both + expect(router.resolveHref('https://site.example.com/pricing')).toBe( + 'https://site.example.com/pricing' + ) + }) + }) + describe('on the site host', () => { it('keeps the /apps prefix', () => { const router = setup({ cookieHostname: 'site.example.com' }) diff --git a/platform/lib/app.action.ts b/platform/lib/app.action.ts index 7a3f2d3..f6552f4 100644 --- a/platform/lib/app.action.ts +++ b/platform/lib/app.action.ts @@ -14,12 +14,12 @@ import { } from '@/lib/app.context' import { APP_AUDIENCE } from '@/lib/audience.consts' import { setupHeadersContext } from '@/lib/context.setup' +import { captureUnknownException } from '@/lib/response' import { getContextFrontendHost, getContextRequestHost, runInContext, } from '@/lib/context.store' -import { captureException } from '@/lib/error' import type { ZodSchema } from '@/lib/zod.schema' import schema from '@/lib/zod.schema' @@ -107,7 +107,9 @@ export function appActionHandler( return it } catch (e) { - await captureException(e) + // @note an error with a known code - not found, not authorized, limits + // reached - is an expected answer the caller renders, not a fault + await captureUnknownException(e) return { error: { diff --git a/platform/lib/app.action.utest.js b/platform/lib/app.action.utest.js index a1fa738..3c4e17f 100644 --- a/platform/lib/app.action.utest.js +++ b/platform/lib/app.action.utest.js @@ -8,7 +8,7 @@ import { getContextFrontendHost, getContextRequestHost, } from '@/lib/context.store' -import { captureException } from '@/lib/error' +import { captureUnknownException } from '@/lib/response' import schema from '@/lib/zod.schema' import { @@ -47,8 +47,8 @@ jest.mock('@/lib/context.setup', () => ({ setupHeadersContext: jest.fn(), })) -jest.mock('@/lib/error', () => ({ - captureException: jest.fn(), +jest.mock('@/lib/response', () => ({ + captureUnknownException: jest.fn(), })) describe('app.action', () => { @@ -195,7 +195,7 @@ describe('app.action', () => { message: 'Test error', }, }) - expect(captureException).toHaveBeenCalledWith(mockError) + expect(captureUnknownException).toHaveBeenCalledWith(mockError) }) it('should handle validation errors in input schema', async () => { @@ -216,7 +216,7 @@ describe('app.action', () => { const result = await handler({ name: 123 }) expect(result).toHaveProperty('error') - expect(captureException).toHaveBeenCalled() + expect(captureUnknownException).toHaveBeenCalled() }) it('should handle validation errors in config schema', async () => { @@ -237,7 +237,7 @@ describe('app.action', () => { const result = await handler({}) expect(result).toHaveProperty('error') - expect(captureException).toHaveBeenCalled() + expect(captureUnknownException).toHaveBeenCalled() }) }) diff --git a/platform/lib/highlighter.ts b/platform/lib/highlighter.ts index 7d868a3..1fa9bda 100644 --- a/platform/lib/highlighter.ts +++ b/platform/lib/highlighter.ts @@ -9,10 +9,18 @@ let highlighterPromise: Promise> | null = export function getHighlighter() { if (!highlighterPromise) { - highlighterPromise = import('shiki').then(({ createHighlighter }) => + // @note the JavaScript engine rather than the default Oniguruma one: the + // latter needs WebAssembly, which iOS Lockdown Mode removes, and the page + // then fails with an unhandled rejection. Forgiving so a grammar the + // engine cannot compile degrades to plain text instead of throwing. + highlighterPromise = Promise.all([ + import('shiki'), + import('shiki/engine/javascript'), + ]).then(([{ createHighlighter }, { createJavaScriptRegexEngine }]) => createHighlighter({ themes: [githubDark, githubLight], langs: [], + engine: createJavaScriptRegexEngine({ forgiving: true }), }) ) } diff --git a/platform/lib/highlighter.utest.js b/platform/lib/highlighter.utest.js index 908c24b..0e4a0e5 100644 --- a/platform/lib/highlighter.utest.js +++ b/platform/lib/highlighter.utest.js @@ -11,7 +11,13 @@ describe('getHighlighter', () => { const mockHighlighter = { codeToHtml: jest.fn() } const createHighlighter = jest.fn().mockResolvedValue(mockHighlighter) + const engine = { name: 'javascript' } + const createJavaScriptRegexEngine = jest.fn(() => engine) + jest.doMock('shiki', () => ({ createHighlighter })) + jest.doMock('shiki/engine/javascript', () => ({ + createJavaScriptRegexEngine, + })) const { getHighlighter: getTestHighlighter } = require('./highlighter') @@ -21,9 +27,13 @@ describe('getHighlighter', () => { expect(first).toBe(second) await expect(first).resolves.toBe(mockHighlighter) expect(createHighlighter).toHaveBeenCalledTimes(1) + expect(createJavaScriptRegexEngine).toHaveBeenCalledWith({ + forgiving: true, + }) expect(createHighlighter).toHaveBeenCalledWith({ themes: [expect.anything(), expect.anything()], langs: [], + engine, }) }) }) diff --git a/platform/pages/api/v1/bot/[botId]/realtime/websocket/create.js b/platform/pages/api/v1/bot/[botId]/realtime/websocket/create.js index 831580b..7f61468 100644 --- a/platform/pages/api/v1/bot/[botId]/realtime/websocket/create.js +++ b/platform/pages/api/v1/bot/[botId]/realtime/websocket/create.js @@ -15,12 +15,12 @@ import { bypassCache } from '@/lib/cache' import { ensureUntrustedContact } from '@/lib/contact.create' import { createConversation } from '@/lib/conversation.create' import debug, { assert, createSpan } from '@/lib/debug' -import { captureError } from '@/lib/error' import schema, { withSchema } from '@/lib/joi.handler' import { withLimits } from '@/lib/limit.handler' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' import { + captureUnknownError, notFound, ok, respondFromError, @@ -295,7 +295,7 @@ export default withPost( } catch (e) { debug(`responding with error`, { e }) - await captureError(e) + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/bot/[botId]/session/create.js b/platform/pages/api/v1/bot/[botId]/session/create.js index c7ca206..fab87e9 100644 --- a/platform/pages/api/v1/bot/[botId]/session/create.js +++ b/platform/pages/api/v1/bot/[botId]/session/create.js @@ -15,12 +15,12 @@ import { bypassCache } from '@/lib/cache' import { ensureUntrustedContact } from '@/lib/contact.create' import { createConversation } from '@/lib/conversation.create' import debug, { assert, createSpan } from '@/lib/debug' -import { captureError } from '@/lib/error' import schema, { withSchema } from '@/lib/joi.handler' import { withLimits } from '@/lib/limit.handler' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' import { + captureUnknownError, notFound, ok, respondFromError, @@ -345,7 +345,7 @@ export default withPost( } catch (e) { debug(`responding with error`, { e }) - await captureError(e) + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/conversation/[conversationId]/_complete.utest.js b/platform/pages/api/v1/conversation/[conversationId]/_complete.utest.js index 4e1a9bc..940c698 100644 --- a/platform/pages/api/v1/conversation/[conversationId]/_complete.utest.js +++ b/platform/pages/api/v1/conversation/[conversationId]/_complete.utest.js @@ -191,7 +191,8 @@ describe('complete', () => { }, }) ) - expect(captureError).toHaveBeenCalledWith(safeError) + // @note a known-code error is the client's answer, so it is not captured + expect(captureError).not.toHaveBeenCalled() }) it('should stream TAG_ERROR when complete throws after successful send', async () => { diff --git a/platform/pages/api/v1/conversation/[conversationId]/apply.js b/platform/pages/api/v1/conversation/[conversationId]/apply.js index 76b8637..b0df5b9 100644 --- a/platform/pages/api/v1/conversation/[conversationId]/apply.js +++ b/platform/pages/api/v1/conversation/[conversationId]/apply.js @@ -6,16 +6,13 @@ import { getStatefulConversationEngine } from '@/lib/conversation.engine' import { TAG_ERROR, TAG_RESULT, createSinkEvent } from '@/lib/conversation.tag' import debug from '@/lib/debug' import { withStreamContinuity } from '@/lib/stream' -import { - captureError, - errorResponseToError, - errorToSafeErrorResponse, -} from '@/lib/error' +import { errorResponseToError, errorToSafeErrorResponse } from '@/lib/error' import { events } from '@/lib/it' import schema, { withSchema } from '@/lib/joi.handler' import { withSessionLimits } from '@/lib/limit.handler' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' +import { captureUnknownError } from '@/lib/response' import extensionsSchema from '@/schemas/inlineExtensions' import functionsSchema from '@/schemas/functionsSchema' @@ -211,7 +208,7 @@ export async function* apply(session, conversationId, body, options = {}) { 'api.v1.conversation.[conversationId].apply' ) - await captureError(e) + await captureUnknownError(e) push( createSinkEvent({ diff --git a/platform/pages/api/v1/conversation/[conversationId]/complete.js b/platform/pages/api/v1/conversation/[conversationId]/complete.js index 8f8ed79..311bc9d 100644 --- a/platform/pages/api/v1/conversation/[conversationId]/complete.js +++ b/platform/pages/api/v1/conversation/[conversationId]/complete.js @@ -12,11 +12,7 @@ import { } from '@/lib/conversation.tag' import debug from '@/lib/debug' import { withStreamContinuity } from '@/lib/stream' -import { - captureError, - errorResponseToError, - errorToSafeErrorResponse, -} from '@/lib/error' +import { errorResponseToError, errorToSafeErrorResponse } from '@/lib/error' import { anySignal } from '@/lib/fetch' import { events } from '@/lib/it' import schema, { withSchema } from '@/lib/joi.handler' @@ -25,6 +21,7 @@ import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' import { getRandomId } from '@/lib/string' import { createTimeoutMonitor } from '@/lib/timeout.monitor' +import { captureUnknownError } from '@/lib/response' import extensionsSchema from '@/schemas/inlineExtensions' import functionsSchema from '@/schemas/functionsSchema' @@ -275,7 +272,7 @@ export async function* complete(session, conversationId, body, options = {}) { 'api.v1.conversation.[conversationId].complete' ) - await captureError(e) + await captureUnknownError(e) push( createSinkEvent({ diff --git a/platform/pages/api/v1/conversation/[conversationId]/message/[messageId]/_update.utest.js b/platform/pages/api/v1/conversation/[conversationId]/message/[messageId]/_update.utest.js index 82bf53d..e86bde4 100644 --- a/platform/pages/api/v1/conversation/[conversationId]/message/[messageId]/_update.utest.js +++ b/platform/pages/api/v1/conversation/[conversationId]/message/[messageId]/_update.utest.js @@ -264,6 +264,60 @@ describe('/api/v1/conversation/[conversationId]/message/[messageId]/update', () }) describe('authorization', () => { + it('should return 400 when switching to activity without activity meta', async () => { + prisma.conversation.findUnique.mockResolvedValue(mockConversation) + + const result = await handler(mockReq, mockSession, { type: 'activity' }) + + expect(result.status).toBe(400) + expect(prisma.message.update).not.toHaveBeenCalled() + }) + + it('should return 400 when replacing the meta of an activity message', async () => { + prisma.conversation.findUnique.mockResolvedValue({ + ...mockConversation, + messages: [ + { + id: 'msg_xyz', + type: 'activity', + meta: { + activity: { type: 'request', function: { name: 'lookup' } }, + }, + }, + ], + }) + + // @note the mocked getMeta merges, so make the merge drop the activity + getMeta.mockReturnValueOnce({ note: 'edited' }) + + const result = await handler(mockReq, mockSession, { + meta: { note: 'edited' }, + }) + + expect(result.status).toBe(400) + expect(prisma.message.update).not.toHaveBeenCalled() + }) + + it('should update the text of an activity message without meta in the body', async () => { + prisma.conversation.findUnique.mockResolvedValue({ + ...mockConversation, + messages: [ + { + id: 'msg_xyz', + type: 'activity', + meta: { + activity: { type: 'request', function: { name: 'lookup' } }, + }, + }, + ], + }) + + const result = await handler(mockReq, mockSession, { text: 'edited' }) + + expect(result.status).toBe(200) + expect(prisma.message.update).toHaveBeenCalled() + }) + it('should return 404 when the conversation does not exist', async () => { prisma.conversation.findUnique.mockResolvedValue(null) diff --git a/platform/pages/api/v1/conversation/[conversationId]/message/[messageId]/update.js b/platform/pages/api/v1/conversation/[conversationId]/message/[messageId]/update.js index 3d81bf6..a009928 100644 --- a/platform/pages/api/v1/conversation/[conversationId]/message/[messageId]/update.js +++ b/platform/pages/api/v1/conversation/[conversationId]/message/[messageId]/update.js @@ -8,27 +8,21 @@ import { getMeta } from '@/lib/meta' import { withPost } from '@/lib/method' import { detectPiiEntities, getSafeTextAndEntities } from '@/lib/pii' import { requiredUrlParam } from '@/lib/query.get' -import { notAuthorized, notFound, ok } from '@/lib/response' +import { badRequest, notAuthorized, notFound, ok } from '@/lib/response' import { withSession } from '@/lib/session.handler' -import descriptionSchema from '@/schemas/description' -import messageTextSchema from '@/schemas/messageText' -import messageTypeSchema from '@/schemas/messageType' -import metaSchema from '@/schemas/meta' -import nameSchema from '@/schemas/name' +import { + assertActivityMessage, + messageFieldsSchema, +} from '@/schemas/messages' -export const bodySchema = schema.object({ - name: nameSchema, - description: descriptionSchema, - - type: messageTypeSchema, - - text: messageTextSchema, - - entities: schema.array().items(schema.object({}).unknown(true)), - - meta: metaSchema, -}) +// @note the message fields with type and text optional - the activity rule +// runs in the handler against the stored message, since the body is partial +export const bodySchema = messageFieldsSchema + .fork(['type', 'text'], (field) => field.optional()) + .append({ + entities: schema.array().items(schema.object({}).unknown(true)), + }) /** * @swagger @@ -135,6 +129,8 @@ export default withPost( select: { id: true, + type: true, + meta: true, }, @@ -155,6 +151,22 @@ export default withPost( return notFound() } + const [existingMessage] = conversation.messages + + const nextMeta = getMeta(meta, existingMessage.meta) + + // @note the update may switch the type to activity or replace the meta, + // and an activity message without its meta breaks the next completion - + // check the message as it will be stored, not the body on its own + try { + assertActivityMessage({ + type: type || existingMessage.type, + meta: meta === undefined ? existingMessage.meta : nextMeta, + }) + } catch (e) { + return badRequest(e.message) + } + await prisma.message.update({ where: { id: conversation.messages[0].id, @@ -174,7 +186,7 @@ export default withPost( // meta and others - meta: getMeta(meta, conversation.messages[0].meta), + meta: nextMeta, }, }) diff --git a/platform/pages/api/v1/conversation/[conversationId]/message/_create.utest.js b/platform/pages/api/v1/conversation/[conversationId]/message/_create.utest.js index 2ad66a6..cb32597 100644 --- a/platform/pages/api/v1/conversation/[conversationId]/message/_create.utest.js +++ b/platform/pages/api/v1/conversation/[conversationId]/message/_create.utest.js @@ -255,32 +255,51 @@ describe('POST /api/v1/conversation/{conversationId}/message/create', () => { }) describe('bodySchema', () => { - it('should require type field', () => { - const { error } = bodySchema.validate({ text: 'hello' }) + // @note the schema carries the activity rule as an external rule, so it + // validates asynchronously + const invalid = (body) => bodySchema.validateAsync(body) - expect(error).toBeDefined() - }) + const valid = async (body) => { + await expect(bodySchema.validateAsync(body)).resolves.toBeDefined() + } - it('should require text field', () => { - const { error } = bodySchema.validate({ type: 'user' }) + it('should require type field', async () => { + await expect(invalid({ text: 'hello' })).rejects.toThrow('"type"') + }) - expect(error).toBeDefined() + it('should require text field', async () => { + await expect(invalid({ type: 'user' })).rejects.toThrow('"text"') }) - it('should accept valid user message', () => { - const { error } = bodySchema.validate({ type: 'user', text: 'hello' }) + it('should accept valid user message', async () => { + await valid({ type: 'user', text: 'hello' }) + }) - expect(error).toBeUndefined() + it('should reject an activity message without activity meta', async () => { + await expect(invalid({ type: 'activity', text: '' })).rejects.toThrow( + "missing 'meta'" + ) }) - it('should accept valid bot message', () => { - const { error } = bodySchema.validate({ type: 'bot', text: 'response' }) + it('should accept an activity message with activity meta', async () => { + await valid({ + type: 'activity', + text: '', + meta: { + activity: { + type: 'request', + function: { name: 'lookup', arguments: {} }, + }, + }, + }) + }) - expect(error).toBeUndefined() + it('should accept valid bot message', async () => { + await valid({ type: 'bot', text: 'response' }) }) - it('should accept all optional fields', () => { - const { error } = bodySchema.validate({ + it('should accept all optional fields', async () => { + await valid({ type: 'user', text: 'hello', name: 'msg', @@ -288,8 +307,6 @@ describe('POST /api/v1/conversation/{conversationId}/message/create', () => { entities: [], meta: {}, }) - - expect(error).toBeUndefined() }) }) }) diff --git a/platform/pages/api/v1/conversation/[conversationId]/message/create.js b/platform/pages/api/v1/conversation/[conversationId]/message/create.js index a85e257..6196388 100644 --- a/platform/pages/api/v1/conversation/[conversationId]/message/create.js +++ b/platform/pages/api/v1/conversation/[conversationId]/message/create.js @@ -11,23 +11,12 @@ import { requiredUrlParam } from '@/lib/query.get' import { notAuthorized, notFound, ok } from '@/lib/response' import { recordMessageUsage } from '@/lib/usage.record' -import descriptionSchema from '@/schemas/description' -import messageTextSchema from '@/schemas/messageText' -import messageTypeSchema from '@/schemas/messageType' -import metaSchema from '@/schemas/meta' -import nameSchema from '@/schemas/name' - -export const bodySchema = schema.object({ - name: nameSchema, - description: descriptionSchema, - - type: messageTypeSchema.required(), - - text: messageTextSchema.required(), +import { messageSchema } from '@/schemas/messages' +// @note the same shape a stateless completion accepts for its messages, so an +// activity message is held to its meta here too +export const bodySchema = messageSchema.append({ entities: schema.array().items(schema.object({}).unknown(true)), - - meta: metaSchema, }) /** diff --git a/platform/pages/api/v1/conversation/[conversationId]/queue.js b/platform/pages/api/v1/conversation/[conversationId]/queue.js index 5cd3c26..2c3e2d2 100644 --- a/platform/pages/api/v1/conversation/[conversationId]/queue.js +++ b/platform/pages/api/v1/conversation/[conversationId]/queue.js @@ -21,7 +21,7 @@ import { createSinkEvent, } from '@/lib/conversation.tag' import debug from '@/lib/debug' -import { captureError, captureInputError } from '@/lib/error' +import { captureInputError } from '@/lib/error' import { ABORT_ERROR_NAME, anySignal } from '@/lib/fetch' import { setupFrontendHostContext } from '@/lib/integration.context' import { tryParse as tryJsonParse } from '@/lib/json' @@ -35,6 +35,7 @@ import { updateSessionStore } from '@/lib/session.context' import { getRandomId } from '@/lib/string' import { userToSessionUser } from '@/lib/user.session' import { parseAsync } from '@/lib/zod.schema' +import { captureUnknownError } from '@/lib/response' import { IDLE_EVENT_TYPE as EXTRACT_INTEGRATION_IDLE_EVENT_TYPE, @@ -812,7 +813,7 @@ export async function handleCompleteEvent(conversationId, payload, context) { 'api.v1.conversation.conversationId.handleCompleteEvent' ) - await captureError(e) + await captureUnknownError(e) await publishChannelMessage( sessionChannelId, diff --git a/platform/pages/api/v1/conversation/[conversationId]/receive.js b/platform/pages/api/v1/conversation/[conversationId]/receive.js index e88d193..7c37870 100644 --- a/platform/pages/api/v1/conversation/[conversationId]/receive.js +++ b/platform/pages/api/v1/conversation/[conversationId]/receive.js @@ -4,15 +4,12 @@ import { getStatefulConversationEngine } from '@/lib/conversation.engine' import { TAG_ERROR, TAG_RESULT, createSinkEvent } from '@/lib/conversation.tag' import debug from '@/lib/debug' import { withStream } from '@/lib/stream' -import { - captureError, - errorResponseToError, - errorToSafeErrorResponse, -} from '@/lib/error' +import { errorResponseToError, errorToSafeErrorResponse } from '@/lib/error' import schema, { withSchema } from '@/lib/joi.handler' import { withSessionLimits } from '@/lib/limit.handler' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' +import { captureUnknownError } from '@/lib/response' import extensionsSchema from '@/schemas/inlineExtensions' import functionsSchema from '@/schemas/functionsSchema' @@ -241,7 +238,7 @@ export default withPost( } catch (e) { debug(`responding with error`, { e }) - await captureError(e) + await captureUnknownError(e) const event = createSinkEvent({ type: TAG_ERROR, diff --git a/platform/pages/api/v1/conversation/[conversationId]/send.js b/platform/pages/api/v1/conversation/[conversationId]/send.js index b6a30dd..3207ea8 100644 --- a/platform/pages/api/v1/conversation/[conversationId]/send.js +++ b/platform/pages/api/v1/conversation/[conversationId]/send.js @@ -4,15 +4,12 @@ import { getStatefulConversationEngine } from '@/lib/conversation.engine' import { TAG_ERROR, TAG_RESULT, createSinkEvent } from '@/lib/conversation.tag' import debug from '@/lib/debug' import { withStream } from '@/lib/stream' -import { - captureError, - errorResponseToError, - errorToSafeErrorResponse, -} from '@/lib/error' +import { errorResponseToError, errorToSafeErrorResponse } from '@/lib/error' import schema, { withSchema } from '@/lib/joi.handler' import { withSessionLimits } from '@/lib/limit.handler' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' +import { captureUnknownError } from '@/lib/response' import extensionsSchema from '@/schemas/inlineExtensions' import functionsSchema from '@/schemas/functionsSchema' @@ -275,7 +272,7 @@ export default withPost( } catch (e) { debug(`responding with error`, { e }) - await captureError(e) + await captureUnknownError(e) const event = createSinkEvent({ type: TAG_ERROR, diff --git a/platform/pages/api/v1/conversation/_complete.utest.js b/platform/pages/api/v1/conversation/_complete.utest.js index f638514..65fbbfc 100644 --- a/platform/pages/api/v1/conversation/_complete.utest.js +++ b/platform/pages/api/v1/conversation/_complete.utest.js @@ -341,7 +341,8 @@ describe('complete', () => { }, }) ) - expect(captureError).toHaveBeenCalledWith(safeError) + // @note a known-code error is the client's answer, so it is not captured + expect(captureError).not.toHaveBeenCalled() }) it('should emit TAG_ERROR and no sendResult when process() throws', async () => { diff --git a/platform/pages/api/v1/conversation/_create.utest.js b/platform/pages/api/v1/conversation/_create.utest.js index 91ab610..47b4601 100644 --- a/platform/pages/api/v1/conversation/_create.utest.js +++ b/platform/pages/api/v1/conversation/_create.utest.js @@ -29,6 +29,8 @@ jest.mock('@/lib/error', () => ({ })) jest.mock('@/lib/response', () => ({ + // @note the real filter, so a known-code error is not captured + captureUnknownError: jest.requireActual('@/lib/response').captureUnknownError, ok: jest.fn((data) => new Response(JSON.stringify(data), { status: 200 })), respondFromError: jest.fn( (e) => new Response(JSON.stringify({ message: e.message }), { status: 500 }) diff --git a/platform/pages/api/v1/conversation/apply.js b/platform/pages/api/v1/conversation/apply.js index df98eda..62999e6 100644 --- a/platform/pages/api/v1/conversation/apply.js +++ b/platform/pages/api/v1/conversation/apply.js @@ -12,11 +12,7 @@ import { } from '@/lib/conversation.tag' import debug from '@/lib/debug' import { withStreamContinuity } from '@/lib/stream' -import { - captureError, - errorResponseToError, - errorToSafeErrorResponse, -} from '@/lib/error' +import { errorResponseToError, errorToSafeErrorResponse } from '@/lib/error' import { events } from '@/lib/it' import schema, { withSchema } from '@/lib/joi.handler' import { withSessionLimits } from '@/lib/limit.handler' @@ -26,7 +22,7 @@ import { uploadNamespaceAttachmentFromURL, } from '@/lib/namespace.attachment' import { getSafeNamespace } from '@/lib/namespace.safe' -import { throwBadRequest } from '@/lib/response' +import { captureUnknownError, throwBadRequest } from '@/lib/response' import { getMaxFileSize } from '@/lib/user.limits' import backstorySchema from '@/schemas/backstory' @@ -346,7 +342,7 @@ export async function* apply(session, body, options = {}) { } catch (e) { debug(`responding with error`, { e }).log('api.v1.conversation.apply') - await captureError(e) + await captureUnknownError(e) push( createSinkEvent({ diff --git a/platform/pages/api/v1/conversation/complete.js b/platform/pages/api/v1/conversation/complete.js index 4a3fb60..a76031a 100644 --- a/platform/pages/api/v1/conversation/complete.js +++ b/platform/pages/api/v1/conversation/complete.js @@ -14,11 +14,7 @@ import { } from '@/lib/conversation.tag' import debug from '@/lib/debug' import { withStreamContinuity } from '@/lib/stream' -import { - captureError, - errorResponseToError, - errorToSafeErrorResponse, -} from '@/lib/error' +import { errorResponseToError, errorToSafeErrorResponse } from '@/lib/error' import { anySignal } from '@/lib/fetch' import { events } from '@/lib/it' import schema, { withSchema } from '@/lib/joi.handler' @@ -29,7 +25,7 @@ import { uploadNamespaceAttachmentFromURL, } from '@/lib/namespace.attachment' import { getSafeNamespace } from '@/lib/namespace.safe' -import { throwBadRequest } from '@/lib/response' +import { captureUnknownError, throwBadRequest } from '@/lib/response' import { createTimeoutMonitor } from '@/lib/timeout.monitor' import { getMaxFileSize } from '@/lib/user.limits' @@ -399,7 +395,7 @@ export async function* complete(session, body, options = {}) { } catch (e) { debug(`responding with error`, { e }).log('api.v1.conversation.complete') - await captureError(e) + await captureUnknownError(e) push( createSinkEvent({ diff --git a/platform/pages/api/v1/conversation/create.js b/platform/pages/api/v1/conversation/create.js index 7311ebd..0b959fb 100644 --- a/platform/pages/api/v1/conversation/create.js +++ b/platform/pages/api/v1/conversation/create.js @@ -2,11 +2,10 @@ import { ensureTrustedContact } from '@/lib/contact.create' import { createConversation } from '@/lib/conversation.create' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import schema, { withSchema } from '@/lib/joi.handler' import { withSessionLimits } from '@/lib/limit.handler' import { withPost } from '@/lib/method' -import { ok, respondFromError } from '@/lib/response' +import { captureUnknownError, ok, respondFromError } from '@/lib/response' import botConfigSchema from '@/schemas/botConfig' import botIdSchema from '@/schemas/botId' @@ -240,7 +239,7 @@ export default withPost( } catch (e) { debug(`responding with error`, { e }) - await captureError(e) + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/conversation/queue.js b/platform/pages/api/v1/conversation/queue.js index 70504f3..917514a 100644 --- a/platform/pages/api/v1/conversation/queue.js +++ b/platform/pages/api/v1/conversation/queue.js @@ -15,7 +15,7 @@ import { } from '@/lib/conversation.idle' import { TAG_ERROR } from '@/lib/conversation.tag' import debug, { assert } from '@/lib/debug' -import { captureError, captureInputError } from '@/lib/error' +import { captureInputError } from '@/lib/error' import { setupFrontendHostContext } from '@/lib/integration.context' import it from '@/lib/it' import { runTasksEach } from '@/lib/job' @@ -25,6 +25,7 @@ import { withQueueHandler } from '@/lib/queue2' import { updateSessionStore } from '@/lib/session.context' import { fastGetUserById } from '@/lib/user.get' import { parseAsync } from '@/lib/zod.schema' +import { captureUnknownError } from '@/lib/response' import { IDLE_EVENT_TYPE as CONVERSATION_IDLE_EVENT_TYPE, @@ -188,7 +189,7 @@ export async function handleCompleteEvent(payload, context) { 'api.v1.conversation.handleCompleteEvent' ) - await captureError(e) + await captureUnknownError(e) await publishChannelMessage( sessionChannelId, diff --git a/platform/pages/api/v1/dataset/[datasetId]/record/[recordId]/_update.utest.js b/platform/pages/api/v1/dataset/[datasetId]/record/[recordId]/_update.utest.js index daac717..ebe630f 100644 --- a/platform/pages/api/v1/dataset/[datasetId]/record/[recordId]/_update.utest.js +++ b/platform/pages/api/v1/dataset/[datasetId]/record/[recordId]/_update.utest.js @@ -35,6 +35,8 @@ jest.mock('@/lib/joi.handler', () => ({ })) jest.mock('@/lib/response', () => ({ + // @note the real filter, so a known-code error is not captured + captureUnknownError: jest.requireActual('@/lib/response').captureUnknownError, notFound: () => ({ status: 404 }), notAuthorized: () => ({ status: 403 }), ok: (data) => ({ status: 200, ...data }), @@ -217,5 +219,12 @@ describe('POST /api/v1/dataset/{datasetId}/record/{recordId}/update', () => { expect(error).toBeUndefined() }) + + it('should reject empty or blank text', () => { + expect(bodySchema.validate({ text: '' }).error.message).toContain('"text"') + expect(bodySchema.validate({ text: ' \n ' }).error.message).toContain( + '"text"' + ) + }) }) }) diff --git a/platform/pages/api/v1/dataset/[datasetId]/record/[recordId]/update.js b/platform/pages/api/v1/dataset/[datasetId]/record/[recordId]/update.js index 0189a84..cc72678 100644 --- a/platform/pages/api/v1/dataset/[datasetId]/record/[recordId]/update.js +++ b/platform/pages/api/v1/dataset/[datasetId]/record/[recordId]/update.js @@ -2,13 +2,18 @@ import prisma from '@/prisma/client' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import schema, { withSchema } from '@/lib/joi.handler' import { getMeta } from '@/lib/meta' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' import { updateRecord } from '@/lib/record' -import { notAuthorized, notFound, ok, respondFromError } from '@/lib/response' +import { + captureUnknownError, + notAuthorized, + notFound, + ok, + respondFromError, +} from '@/lib/response' import { withSession } from '@/lib/session.handler' import { getStore } from '@/lib/store.types' @@ -17,7 +22,9 @@ import recordTextSchema from '@/schemas/recordText' import sourceSchema from '@/schemas/source' export const bodySchema = schema.object({ - text: recordTextSchema, + // @note optional, but the store keeps the stored text only when the field + // is absent - an empty string would replace it and the store refuses that + text: recordTextSchema.invalid('').pattern(/\S/, 'non-blank'), source: sourceSchema, @@ -123,7 +130,7 @@ export default withPost( return ok({ id: recordId }) } catch (e) { - await captureError(e) + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/dataset/[datasetId]/record/_create.utest.js b/platform/pages/api/v1/dataset/[datasetId]/record/_create.utest.js index 43fe404..fff5720 100644 --- a/platform/pages/api/v1/dataset/[datasetId]/record/_create.utest.js +++ b/platform/pages/api/v1/dataset/[datasetId]/record/_create.utest.js @@ -7,7 +7,7 @@ import { captureError } from '@/lib/error' import { createRecord } from '@/lib/record' import { getStore } from '@/lib/store.types' -import handler from './create' +import handler, { bodySchema } from './create' jest.mock('@/prisma/client', () => ({ __esModule: true, @@ -30,23 +30,19 @@ jest.mock('@/lib/limit.handler', () => ({ withSessionLimits: (_limits, fn) => fn, })) -jest.mock('@/lib/joi.handler', () => { - const schema = { - object: jest.fn().mockReturnThis(), - } - - return { - __esModule: true, - default: schema, - withSchema: jest.fn((_schema, fn) => fn), - } -}) +jest.mock('@/lib/joi.handler', () => ({ + __esModule: true, + ...jest.requireActual('@/lib/joi.handler'), + withSchema: jest.fn((_schema, fn) => fn), +})) jest.mock('@/lib/query.get', () => ({ requiredUrlParam: jest.fn((req, param) => req.query[param]), })) jest.mock('@/lib/response', () => ({ + // @note the real filter, so a known-code error is not captured + captureUnknownError: jest.requireActual('@/lib/response').captureUnknownError, ok: (data) => ({ status: 200, body: data }), notFound: () => ({ status: 404 }), notAuthorized: () => ({ status: 403 }), @@ -211,4 +207,21 @@ describe('POST /api/v1/dataset/[datasetId]/record/create', () => { ) }) }) + + describe('bodySchema', () => { + it('should accept text', () => { + expect(bodySchema.validate({ text: 'Some record text' }).error).toBeUndefined() + }) + + it('should require text', () => { + expect(bodySchema.validate({}).error.message).toContain('"text"') + }) + + it('should reject empty or blank text', () => { + expect(bodySchema.validate({ text: '' }).error.message).toContain('"text"') + expect(bodySchema.validate({ text: ' \n\t ' }).error.message).toContain( + '"text"' + ) + }) + }) }) diff --git a/platform/pages/api/v1/dataset/[datasetId]/record/create.js b/platform/pages/api/v1/dataset/[datasetId]/record/create.js index c61a00d..b6e3b1f 100644 --- a/platform/pages/api/v1/dataset/[datasetId]/record/create.js +++ b/platform/pages/api/v1/dataset/[datasetId]/record/create.js @@ -2,13 +2,18 @@ import prisma from '@/prisma/client' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import schema, { withSchema } from '@/lib/joi.handler' import { withSessionLimits } from '@/lib/limit.handler' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' import { createRecord } from '@/lib/record' -import { notAuthorized, notFound, ok, respondFromError } from '@/lib/response' +import { + captureUnknownError, + notAuthorized, + notFound, + ok, + respondFromError, +} from '@/lib/response' import { getStore } from '@/lib/store.types' import metaSchema from '@/schemas/meta' @@ -16,7 +21,9 @@ import recordTextSchema from '@/schemas/recordText' import sourceSchema from '@/schemas/source' export const bodySchema = schema.object({ - text: recordTextSchema.required(), + // @note the vector store refuses a record without text - reject blank text + // here so the caller gets a 400 instead of a store error + text: recordTextSchema.invalid('').pattern(/\S/, 'non-blank').required(), source: sourceSchema, @@ -105,7 +112,7 @@ export default withPost( return ok({ id }) } catch (e) { - await captureError(e) + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/file/[fileId]/thumbnail/_download.utest.js b/platform/pages/api/v1/file/[fileId]/thumbnail/_download.utest.js index c926373..198c186 100644 --- a/platform/pages/api/v1/file/[fileId]/thumbnail/_download.utest.js +++ b/platform/pages/api/v1/file/[fileId]/thumbnail/_download.utest.js @@ -65,6 +65,8 @@ jest.mock('@/lib/response', () => ({ })) const { getFileInstance } = require('@/lib/file.storage') +const { createThumbnail } = require('@/lib/image.transform') +const { captureUnknownException } = require('@/lib/response') const { getSession } = require('@/lib/session.get') describe('GET /api/v1/file/[fileId]/thumbnail/download', () => { @@ -116,4 +118,38 @@ describe('GET /api/v1/file/[fileId]/thumbnail/download', () => { }) expect(result.headers).not.toHaveProperty('Vercel-CDN-Cache-Control') }) + + it('serves the icon without capturing when the image does not decode', async () => { + prisma.file.findUnique.mockResolvedValue({ + id: 'file123', + userId: 'user123', + visibility: FileVisibility.public, + meta: { contentType: 'image/png' }, + }) + createThumbnail.mockRejectedValueOnce( + new Error('unrecognised content at end of stream') + ) + + const result = await handler({ query: { fileId: 'file123' } }) + + expect(result.status).toBe(200) + expect(result.headers).toMatchObject({ 'Content-Type': 'image/svg+xml' }) + expect(captureUnknownException).not.toHaveBeenCalled() + }) + + it('captures a storage failure before falling back to the icon', async () => { + prisma.file.findUnique.mockResolvedValue({ + id: 'file123', + userId: 'user123', + visibility: FileVisibility.public, + meta: { contentType: 'image/png' }, + }) + getFileInstance.mockRejectedValueOnce(new Error('storage down')) + + const result = await handler({ query: { fileId: 'file123' } }) + + expect(result.status).toBe(200) + expect(result.headers).toMatchObject({ 'Content-Type': 'image/svg+xml' }) + expect(captureUnknownException).toHaveBeenCalledTimes(1) + }) }) diff --git a/platform/pages/api/v1/file/[fileId]/thumbnail/download.js b/platform/pages/api/v1/file/[fileId]/thumbnail/download.js index dc44b61..cd795f1 100644 --- a/platform/pages/api/v1/file/[fileId]/thumbnail/download.js +++ b/platform/pages/api/v1/file/[fileId]/thumbnail/download.js @@ -109,16 +109,23 @@ export default withGet(async function (req) { } const imageData = await fileInstance.arrayBuffer() - const { buffer: thumbnailBuffer, mimeType } = await createThumbnail( - imageData, - { contentType } - ) - return send(thumbnailBuffer, { - 'Content-Type': mimeType, + let thumbnail - ...(cacheHeaders || null), - }) + try { + thumbnail = await createThumbnail(imageData, { contentType }) + } catch { + // @note the stored bytes are user input - a truncated or mislabeled + // image fails to decode and gets the icon, which is not a bug + } + + if (thumbnail) { + return send(thumbnail.buffer, { + 'Content-Type': thumbnail.mimeType, + + ...(cacheHeaders || null), + }) + } } catch (e) { await captureUnknownException(e) diff --git a/platform/pages/api/v1/hub/bot/[botId]/session/_create.utest.js b/platform/pages/api/v1/hub/bot/[botId]/session/_create.utest.js index 7186aa2..8d2d65a 100644 --- a/platform/pages/api/v1/hub/bot/[botId]/session/_create.utest.js +++ b/platform/pages/api/v1/hub/bot/[botId]/session/_create.utest.js @@ -47,6 +47,8 @@ jest.mock('@/lib/query.get', () => ({ })) jest.mock('@/lib/response', () => ({ + // @note the real filter, so a known-code error is not captured + captureUnknownError: jest.requireActual('@/lib/response').captureUnknownError, ok: (body) => ({ status: 200, body }), throwNotFound: () => ({ status: 404 }), respondFromError: (error) => ({ diff --git a/platform/pages/api/v1/hub/bot/[botId]/session/create.js b/platform/pages/api/v1/hub/bot/[botId]/session/create.js index a8ba359..ebf0d5c 100644 --- a/platform/pages/api/v1/hub/bot/[botId]/session/create.js +++ b/platform/pages/api/v1/hub/bot/[botId]/session/create.js @@ -6,12 +6,16 @@ import prisma from '@/prisma/client' import { getConversationDetails } from '@/lib/bot.conversation' import { createConversation } from '@/lib/conversation.create' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import schema, { withSchema } from '@/lib/joi.handler' import { withLimits } from '@/lib/limit.handler' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' -import { ok, respondFromError, throwNotFound } from '@/lib/response' +import { + captureUnknownError, + ok, + respondFromError, + throwNotFound, +} from '@/lib/response' import { withSession } from '@/lib/session.handler' import { createConversationSessionToken } from '@/pages/api/v1/conversation/[conversationId]/session/create' @@ -97,7 +101,7 @@ export default withPost( } catch (e) { debug(`responding with error`, { e }) - await captureError(e) + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/hub/dataset/[datasetId]/session/_create.utest.js b/platform/pages/api/v1/hub/dataset/[datasetId]/session/_create.utest.js index 9318f36..dbd5909 100644 --- a/platform/pages/api/v1/hub/dataset/[datasetId]/session/_create.utest.js +++ b/platform/pages/api/v1/hub/dataset/[datasetId]/session/_create.utest.js @@ -47,6 +47,8 @@ jest.mock('@/lib/query.get', () => ({ })) jest.mock('@/lib/response', () => ({ + // @note the real filter, so a known-code error is not captured + captureUnknownError: jest.requireActual('@/lib/response').captureUnknownError, ok: (body) => ({ status: 200, body }), throwNotFound: () => ({ status: 404 }), respondFromError: (error) => ({ diff --git a/platform/pages/api/v1/hub/dataset/[datasetId]/session/create.js b/platform/pages/api/v1/hub/dataset/[datasetId]/session/create.js index e9ec8a0..79c6373 100644 --- a/platform/pages/api/v1/hub/dataset/[datasetId]/session/create.js +++ b/platform/pages/api/v1/hub/dataset/[datasetId]/session/create.js @@ -6,12 +6,16 @@ import prisma from '@/prisma/client' import { getConversationDetails } from '@/lib/bot.conversation' import { createConversation } from '@/lib/conversation.create' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import schema, { withSchema } from '@/lib/joi.handler' import { withLimits } from '@/lib/limit.handler' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' -import { ok, respondFromError, throwNotFound } from '@/lib/response' +import { + captureUnknownError, + ok, + respondFromError, + throwNotFound, +} from '@/lib/response' import { withSession } from '@/lib/session.handler' import backstorySchema from '@/schemas/backstory' @@ -111,7 +115,7 @@ export default withPost( } catch (e) { debug(`responding with error`, { e }) - await captureError(e) + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/hub/skillset/[skillsetId]/session/_create.utest.js b/platform/pages/api/v1/hub/skillset/[skillsetId]/session/_create.utest.js index 08449b6..e1de2bd 100644 --- a/platform/pages/api/v1/hub/skillset/[skillsetId]/session/_create.utest.js +++ b/platform/pages/api/v1/hub/skillset/[skillsetId]/session/_create.utest.js @@ -47,6 +47,8 @@ jest.mock('@/lib/query.get', () => ({ })) jest.mock('@/lib/response', () => ({ + // @note the real filter, so a known-code error is not captured + captureUnknownError: jest.requireActual('@/lib/response').captureUnknownError, ok: (body) => ({ status: 200, body }), throwNotFound: () => ({ status: 404 }), respondFromError: (error) => ({ diff --git a/platform/pages/api/v1/hub/skillset/[skillsetId]/session/create.js b/platform/pages/api/v1/hub/skillset/[skillsetId]/session/create.js index ad474f1..6ad68c2 100644 --- a/platform/pages/api/v1/hub/skillset/[skillsetId]/session/create.js +++ b/platform/pages/api/v1/hub/skillset/[skillsetId]/session/create.js @@ -6,12 +6,16 @@ import prisma from '@/prisma/client' import { getConversationDetails } from '@/lib/bot.conversation' import { createConversation } from '@/lib/conversation.create' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import schema, { withSchema } from '@/lib/joi.handler' import { withLimits } from '@/lib/limit.handler' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' -import { ok, respondFromError, throwNotFound } from '@/lib/response' +import { + captureUnknownError, + ok, + respondFromError, + throwNotFound, +} from '@/lib/response' import { withSession } from '@/lib/session.handler' import backstorySchema from '@/schemas/backstory' @@ -111,7 +115,7 @@ export default withPost( } catch (e) { debug(`responding with error`, { e }) - await captureError(e) + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/integration/discord/[discordIntegrationId]/_setup.utest.js b/platform/pages/api/v1/integration/discord/[discordIntegrationId]/_setup.utest.js index 08d830c..fb44648 100644 --- a/platform/pages/api/v1/integration/discord/[discordIntegrationId]/_setup.utest.js +++ b/platform/pages/api/v1/integration/discord/[discordIntegrationId]/_setup.utest.js @@ -357,7 +357,8 @@ describe('POST /api/v1/integration/discord/[discordIntegrationId]/setup', () => const { captureError } = jest.requireMock('@/lib/error') const { respondFromError } = jest.requireMock('@/lib/response') - expect(captureError).toHaveBeenCalled() + // @note a conflict is an expected answer, so it is not captured + expect(captureError).not.toHaveBeenCalled() expect(respondFromError).toHaveBeenCalled() }) }) diff --git a/platform/pages/api/v1/integration/discord/[discordIntegrationId]/setup.js b/platform/pages/api/v1/integration/discord/[discordIntegrationId]/setup.js index 110b170..a46ffea 100644 --- a/platform/pages/api/v1/integration/discord/[discordIntegrationId]/setup.js +++ b/platform/pages/api/v1/integration/discord/[discordIntegrationId]/setup.js @@ -3,10 +3,10 @@ import prisma from '@/prisma/client' import debug from '@/lib/debug' import { fetchAPI } from '@/lib/discord.api' -import { captureError } from '@/lib/error' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' import { + captureUnknownError, notAuthorized, notFound, ok, @@ -153,7 +153,9 @@ export default withPost( try { await doSetup(discordIntegration) } catch (e) { - await captureError(e) + // @note a setup refused for a missing or rejected configuration is a + // conflict the caller reads, not a fault + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/integration/email/[emailIntegrationId]/setup.js b/platform/pages/api/v1/integration/email/[emailIntegrationId]/setup.js index f2e219a..c8aa671 100644 --- a/platform/pages/api/v1/integration/email/[emailIntegrationId]/setup.js +++ b/platform/pages/api/v1/integration/email/[emailIntegrationId]/setup.js @@ -2,10 +2,15 @@ import prisma from '@/prisma/client' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' -import { notAuthorized, notFound, ok, respondFromError } from '@/lib/response' +import { + captureUnknownError, + notAuthorized, + notFound, + ok, + respondFromError, +} from '@/lib/response' import { withSession } from '@/lib/session.handler' /** @@ -74,7 +79,9 @@ export default withPost( try { await doSetup(emailIntegration) } catch (e) { - await captureError(e) + // @note a setup refused for a missing or rejected configuration is a + // conflict the caller reads, not a fault + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/integration/github/[githubIntegrationId]/_queue.utest.js b/platform/pages/api/v1/integration/github/[githubIntegrationId]/_queue.utest.js index 1c484da..e4df878 100644 --- a/platform/pages/api/v1/integration/github/[githubIntegrationId]/_queue.utest.js +++ b/platform/pages/api/v1/integration/github/[githubIntegrationId]/_queue.utest.js @@ -9,9 +9,11 @@ import { mintInstallationToken, postIssueComment, } from '@/lib/github.app' +import { getStatefulConversationEngine } from '@/lib/conversation.engine' import { accountConversationalLimitsOk } from '@/lib/limit.core' import { logEvent } from '@/lib/log' import memcache from '@/lib/memcache' +import { captureUnknownException } from '@/lib/response' import { handleInteractEvent } from '@/pages/api/v1/integration/github/[githubIntegrationId]/queue' @@ -95,7 +97,6 @@ jest.mock('@/lib/conversation.engine', () => ({ })) jest.mock('@/lib/error', () => ({ - captureException: jest.fn(), captureInputError: jest.fn(), })) @@ -113,6 +114,7 @@ jest.mock( ) jest.mock('@/lib/response', () => ({ + captureUnknownException: jest.fn(), throwLimitsReached: jest.fn(() => { throw new Error('limits reached') }), @@ -179,6 +181,28 @@ describe('GitHub queue allowFrom gate', () => { expect(postIssueComment).toHaveBeenCalled() }) + it('logs a reply failure through the expected-error filter', async () => { + // @note a GitHub 4xx reaches here as a FetchError with a known code; the + // filter keeps it out of Sentry while a real fault still gets captured + const upstream = new Error('GitHub API GET /orgs/acme/issues failed: 404') + + getStatefulConversationEngine.mockResolvedValueOnce({ + send: jest.fn(async () => undefined), + receive: jest.fn(async () => { + throw upstream + }), + dispose: jest.fn(async () => undefined), + }) + + await handleInteractEvent(githubIntegrationId, payload()) + + expect(postIssueComment).not.toHaveBeenCalled() + expect(captureUnknownException).toHaveBeenCalledWith(upstream) + expect(logEvent).toHaveBeenCalledWith( + expect.objectContaining({ name: 'GitHub Integration Failed' }) + ) + }) + it('answers a listed login', async () => { mockIntegration('@octocat') diff --git a/platform/pages/api/v1/integration/github/[githubIntegrationId]/queue.js b/platform/pages/api/v1/integration/github/[githubIntegrationId]/queue.js index be6b409..0b4004b 100644 --- a/platform/pages/api/v1/integration/github/[githubIntegrationId]/queue.js +++ b/platform/pages/api/v1/integration/github/[githubIntegrationId]/queue.js @@ -16,7 +16,7 @@ import { createConversation } from '@/lib/conversation.create' import { getStatefulConversationEngine } from '@/lib/conversation.engine' import { hasConversation } from '@/lib/conversation.find' import debug from '@/lib/debug' -import { captureException, captureInputError } from '@/lib/error' +import { captureInputError } from '@/lib/error' import { assertAppCredentials, createCommentReaction, @@ -38,7 +38,11 @@ import { logEvent } from '@/lib/log' import memcache from '@/lib/memcache' import queue from '@/lib/queue' import { withQueueHandlerBounded } from '@/lib/queue2' -import { throwLimitsReached, throwNotFound } from '@/lib/response' +import { + captureUnknownException, + throwLimitsReached, + throwNotFound, +} from '@/lib/response' import { updateSessionStore } from '@/lib/session.context' import { resolveSessionDuration } from '@/lib/session.duration' import { userToSessionUser } from '@/lib/user.session' @@ -575,7 +579,9 @@ You can only reply with comments; do not promise actions beyond commenting. await postIssueComment({ token, owner, repo, issueNumber, body: reply }) } } catch (error) { - await captureException(error) + // @note a GitHub 4xx surfaces here as a FetchError with a known code - + // the agent already sees it, so it is not captured + await captureUnknownException(error) await logEvent({ user: { id: integration.userId }, diff --git a/platform/pages/api/v1/integration/googlechat/[googlechatIntegrationId]/setup.js b/platform/pages/api/v1/integration/googlechat/[googlechatIntegrationId]/setup.js index 6a7969b..a49fb90 100644 --- a/platform/pages/api/v1/integration/googlechat/[googlechatIntegrationId]/setup.js +++ b/platform/pages/api/v1/integration/googlechat/[googlechatIntegrationId]/setup.js @@ -3,11 +3,11 @@ import prisma from '@/prisma/client' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import fetch from '@/lib/fetch' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' import { + captureUnknownError, notAuthorized, notFound, ok, @@ -199,7 +199,9 @@ export default withPost( try { await doSetup(googlechatIntegration) } catch (e) { - await captureError(e) + // @note a setup refused for a missing or rejected configuration is a + // conflict the caller reads, not a fault + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/integration/instagram/[instagramIntegrationId]/_setup.utest.js b/platform/pages/api/v1/integration/instagram/[instagramIntegrationId]/_setup.utest.js index d9bd757..bc72b84 100644 --- a/platform/pages/api/v1/integration/instagram/[instagramIntegrationId]/_setup.utest.js +++ b/platform/pages/api/v1/integration/instagram/[instagramIntegrationId]/_setup.utest.js @@ -299,7 +299,8 @@ describe('POST /api/v1/integration/instagram/[instagramIntegrationId]/setup', () const { captureError } = jest.requireMock('@/lib/error') const { respondFromError } = jest.requireMock('@/lib/response') - expect(captureError).toHaveBeenCalled() + // @note a conflict is an expected answer, so it is not captured + expect(captureError).not.toHaveBeenCalled() expect(respondFromError).toHaveBeenCalled() }) }) diff --git a/platform/pages/api/v1/integration/instagram/[instagramIntegrationId]/setup.js b/platform/pages/api/v1/integration/instagram/[instagramIntegrationId]/setup.js index 780dbec..db31a53 100644 --- a/platform/pages/api/v1/integration/instagram/[instagramIntegrationId]/setup.js +++ b/platform/pages/api/v1/integration/instagram/[instagramIntegrationId]/setup.js @@ -3,11 +3,11 @@ import prisma from '@/prisma/client' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import fetch from '@/lib/fetch' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' import { + captureUnknownError, notAuthorized, notFound, ok, @@ -128,7 +128,9 @@ export default withPost( try { await doSetup(instagramIntegration) } catch (e) { - await captureError(e) + // @note a setup refused for a missing or rejected configuration is a + // conflict the caller reads, not a fault + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/integration/messenger/[messengerIntegrationId]/_setup.utest.js b/platform/pages/api/v1/integration/messenger/[messengerIntegrationId]/_setup.utest.js index 823dffe..c3ade43 100644 --- a/platform/pages/api/v1/integration/messenger/[messengerIntegrationId]/_setup.utest.js +++ b/platform/pages/api/v1/integration/messenger/[messengerIntegrationId]/_setup.utest.js @@ -46,7 +46,12 @@ jest.mock('@/lib/query.get', () => ({ jest.mock('@/lib/error', () => ({ captureError: jest.fn(), - SystemError: class SystemError extends Error {}, + SystemError: class SystemError extends Error { + constructor(message, code) { + super(message) + this.code = code + } + }, })) jest.mock('@/lib/response', () => { @@ -280,7 +285,8 @@ describe('POST /api/v1/integration/messenger/[messengerIntegrationId]/setup', () const { captureError } = jest.requireMock('@/lib/error') const { respondFromError } = jest.requireMock('@/lib/response') - expect(captureError).toHaveBeenCalled() + // @note a conflict is an expected answer, so it is not captured + expect(captureError).not.toHaveBeenCalled() expect(respondFromError).toHaveBeenCalled() }) }) diff --git a/platform/pages/api/v1/integration/messenger/[messengerIntegrationId]/setup.js b/platform/pages/api/v1/integration/messenger/[messengerIntegrationId]/setup.js index 6456069..b555bfb 100644 --- a/platform/pages/api/v1/integration/messenger/[messengerIntegrationId]/setup.js +++ b/platform/pages/api/v1/integration/messenger/[messengerIntegrationId]/setup.js @@ -3,11 +3,11 @@ import prisma from '@/prisma/client' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import fetch from '@/lib/fetch' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' import { + captureUnknownError, notAuthorized, notFound, ok, @@ -150,7 +150,9 @@ export default withPost( try { await doSetup(messengerIntegration) } catch (e) { - await captureError(e) + // @note a setup refused for a missing or rejected configuration is a + // conflict the caller reads, not a fault + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/integration/microsoftteams/[microsoftteamsIntegrationId]/setup.js b/platform/pages/api/v1/integration/microsoftteams/[microsoftteamsIntegrationId]/setup.js index bab9704..b3e3ae6 100644 --- a/platform/pages/api/v1/integration/microsoftteams/[microsoftteamsIntegrationId]/setup.js +++ b/platform/pages/api/v1/integration/microsoftteams/[microsoftteamsIntegrationId]/setup.js @@ -3,11 +3,11 @@ import prisma from '@/prisma/client' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import fetch from '@/lib/fetch' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' import { + captureUnknownError, notAuthorized, notFound, ok, @@ -133,7 +133,9 @@ export default withPost( try { await doSetup(microsoftteamsIntegration) } catch (e) { - await captureError(e) + // @note a setup refused for a missing or rejected configuration is a + // conflict the caller reads, not a fault + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/integration/notion/[notionIntegrationId]/_sync.utest.js b/platform/pages/api/v1/integration/notion/[notionIntegrationId]/_sync.utest.js index 05c6f4d..74a0e88 100644 --- a/platform/pages/api/v1/integration/notion/[notionIntegrationId]/_sync.utest.js +++ b/platform/pages/api/v1/integration/notion/[notionIntegrationId]/_sync.utest.js @@ -127,6 +127,14 @@ describe('doSync (notion integration)', () => { expect(require('@/lib/batch').runBatchJobAsync).not.toHaveBeenCalled() }) + it('should throw conflict when the token is missing', async () => { + const integration = makeIntegration({ token: null }) + + await expect(doSync(integration)).rejects.toThrow('No token specified') + + expect(require('@/lib/batch').runBatchJobAsync).not.toHaveBeenCalled() + }) + it('should return without launching job when database limits are exceeded', async () => { const { databaseLimitsOk } = require('@/lib/limit.core') diff --git a/platform/pages/api/v1/integration/notion/[notionIntegrationId]/sync.js b/platform/pages/api/v1/integration/notion/[notionIntegrationId]/sync.js index e44a0de..d05c679 100644 --- a/platform/pages/api/v1/integration/notion/[notionIntegrationId]/sync.js +++ b/platform/pages/api/v1/integration/notion/[notionIntegrationId]/sync.js @@ -43,6 +43,14 @@ export async function doSync(notionIntegration) { return } + // @note the token is optional at create time, as on every other integration + + if (!notionIntegration.token) { + throwConflict('No token specified') + + return + } + if (!(await databaseLimitsOk(notionIntegration.user, ['database/record']))) { debug(`aborting due to exceeded limits`) @@ -112,8 +120,8 @@ export async function doSync(notionIntegration) { expiresAt: notionIntegration.expiresIn ? Date.now() + notionIntegration.expiresIn : scheduleIn - ? Date.now() + scheduleIn - : undefined, + ? Date.now() + scheduleIn + : undefined, // limits diff --git a/platform/pages/api/v1/integration/sitemap/[sitemapIntegrationId]/update.js b/platform/pages/api/v1/integration/sitemap/[sitemapIntegrationId]/update.js index 41c521c..62fe112 100644 --- a/platform/pages/api/v1/integration/sitemap/[sitemapIntegrationId]/update.js +++ b/platform/pages/api/v1/integration/sitemap/[sitemapIntegrationId]/update.js @@ -14,6 +14,7 @@ import { withSession } from '@/lib/session.handler' import aliasSchema from '@/schemas/alias' import blueprintIdSchema from '@/schemas/blueprintId' import datasetIdSchema from '@/schemas/datasetId' +import dbSourceUrlSchema from '@/schemas/dbSourceUrl' import dbStringSchema from '@/schemas/dbString' import descriptionSchema from '@/schemas/description' import metaSchema from '@/schemas/meta' @@ -30,7 +31,7 @@ export const bodySchema = schema.object({ datasetId: datasetIdSchema('manipulate'), - url: dbStringSchema.uri({ + url: dbSourceUrlSchema.uri({ scheme: ['http', 'https'], }), diff --git a/platform/pages/api/v1/integration/sitemap/create.js b/platform/pages/api/v1/integration/sitemap/create.js index 4f540b1..b273bf2 100644 --- a/platform/pages/api/v1/integration/sitemap/create.js +++ b/platform/pages/api/v1/integration/sitemap/create.js @@ -12,6 +12,7 @@ import { ok } from '@/lib/response' import aliasSchema from '@/schemas/alias' import blueprintIdSchema from '@/schemas/blueprintId' import datasetIdSchema from '@/schemas/datasetId' +import dbSourceUrlSchema from '@/schemas/dbSourceUrl' import dbStringSchema from '@/schemas/dbString' import descriptionSchema from '@/schemas/description' import metaSchema from '@/schemas/meta' @@ -28,7 +29,7 @@ export const bodySchema = schema.object({ datasetId: datasetIdSchema('manipulate'), - url: dbStringSchema.uri({ + url: dbSourceUrlSchema.uri({ scheme: ['http', 'https'], }), diff --git a/platform/pages/api/v1/integration/slack/[slackIntegrationId]/setup.js b/platform/pages/api/v1/integration/slack/[slackIntegrationId]/setup.js index 1b02b16..7e8a5a9 100644 --- a/platform/pages/api/v1/integration/slack/[slackIntegrationId]/setup.js +++ b/platform/pages/api/v1/integration/slack/[slackIntegrationId]/setup.js @@ -3,12 +3,12 @@ import prisma from '@/prisma/client' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import fetch from '@/lib/fetch' import { logEvent } from '@/lib/log' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' import { + captureUnknownError, notAuthorized, notFound, ok, @@ -210,7 +210,9 @@ export default withPost( try { await doSetup(slackIntegration) } catch (e) { - await captureError(e) + // @note a setup refused for a missing or rejected configuration is a + // conflict the caller reads, not a fault + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/integration/trigger/[triggerIntegrationId]/setup.js b/platform/pages/api/v1/integration/trigger/[triggerIntegrationId]/setup.js index 8c4fcf4..829c3a9 100644 --- a/platform/pages/api/v1/integration/trigger/[triggerIntegrationId]/setup.js +++ b/platform/pages/api/v1/integration/trigger/[triggerIntegrationId]/setup.js @@ -2,10 +2,15 @@ import prisma from '@/prisma/client' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' -import { notAuthorized, notFound, ok, respondFromError } from '@/lib/response' +import { + captureUnknownError, + notAuthorized, + notFound, + ok, + respondFromError, +} from '@/lib/response' import { withSession } from '@/lib/session.handler' /** @@ -74,7 +79,9 @@ export default withPost( try { await doSetup(triggerIntegration) } catch (e) { - await captureError(e) + // @note a setup refused for a missing or rejected configuration is a + // conflict the caller reads, not a fault + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/integration/twilio/[twilioIntegrationId]/setup.js b/platform/pages/api/v1/integration/twilio/[twilioIntegrationId]/setup.js index 035901c..9567fd0 100644 --- a/platform/pages/api/v1/integration/twilio/[twilioIntegrationId]/setup.js +++ b/platform/pages/api/v1/integration/twilio/[twilioIntegrationId]/setup.js @@ -2,10 +2,15 @@ import prisma from '@/prisma/client' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' -import { notAuthorized, notFound, ok, respondFromError } from '@/lib/response' +import { + captureUnknownError, + notAuthorized, + notFound, + ok, + respondFromError, +} from '@/lib/response' import { withSession } from '@/lib/session.handler' /** @@ -74,7 +79,9 @@ export default withPost( try { await doSetup(twilioIntegration) } catch (e) { - await captureError(e) + // @note a setup refused for a missing or rejected configuration is a + // conflict the caller reads, not a fault + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/integration/whatsapp/[whatsappIntegrationId]/setup.js b/platform/pages/api/v1/integration/whatsapp/[whatsappIntegrationId]/setup.js index 735d1a5..9d66ed3 100644 --- a/platform/pages/api/v1/integration/whatsapp/[whatsappIntegrationId]/setup.js +++ b/platform/pages/api/v1/integration/whatsapp/[whatsappIntegrationId]/setup.js @@ -2,10 +2,10 @@ import prisma from '@/prisma/client' import debug from '@/lib/debug' -import { captureError } from '@/lib/error' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' import { + captureUnknownError, notAuthorized, notFound, ok, @@ -88,7 +88,9 @@ export default withPost( try { await doSetup(whatsappIntegration) } catch (e) { - await captureError(e) + // @note a setup refused for a missing or rejected configuration is a + // conflict the caller reads, not a fault + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/integration/widget/[widgetIntegrationId]/session/_create.utest.js b/platform/pages/api/v1/integration/widget/[widgetIntegrationId]/session/_create.utest.js index a28b7ca..3aa04e0 100644 --- a/platform/pages/api/v1/integration/widget/[widgetIntegrationId]/session/_create.utest.js +++ b/platform/pages/api/v1/integration/widget/[widgetIntegrationId]/session/_create.utest.js @@ -50,6 +50,8 @@ jest.mock('@/lib/query.get', () => ({ })) jest.mock('@/lib/response', () => ({ + // @note the real filter, so a known-code error is not captured + captureUnknownError: jest.requireActual('@/lib/response').captureUnknownError, ok: (data) => ({ status: 200, body: data }), notFound: () => ({ status: 404 }), respondFromError: (err) => ({ status: 500, error: err }), diff --git a/platform/pages/api/v1/integration/widget/[widgetIntegrationId]/session/create.js b/platform/pages/api/v1/integration/widget/[widgetIntegrationId]/session/create.js index cff3b8f..97214d1 100644 --- a/platform/pages/api/v1/integration/widget/[widgetIntegrationId]/session/create.js +++ b/platform/pages/api/v1/integration/widget/[widgetIntegrationId]/session/create.js @@ -21,12 +21,17 @@ import { import { createConversation } from '@/lib/conversation.create' import cuid from '@/lib/cuid' import debug, { assert, createSpan } from '@/lib/debug' -import { captureError } from '@/lib/error' import schema, { withSchema } from '@/lib/joi.handler' import { withLimits } from '@/lib/limit.handler' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' -import { notFound, ok, respondFromError, throwNotFound } from '@/lib/response' +import { + captureUnknownError, + notFound, + ok, + respondFromError, + throwNotFound, +} from '@/lib/response' import { getRandomId } from '@/lib/string' import { cacheUser, fastGetUserById } from '@/lib/user.get' @@ -335,7 +340,7 @@ export default withPost( } catch (e) { debug(`responding with error`, { e }) - await captureError(e) + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/integration/widget/[widgetIntegrationId]/setup.js b/platform/pages/api/v1/integration/widget/[widgetIntegrationId]/setup.js index 741d19e..1a5281b 100644 --- a/platform/pages/api/v1/integration/widget/[widgetIntegrationId]/setup.js +++ b/platform/pages/api/v1/integration/widget/[widgetIntegrationId]/setup.js @@ -2,11 +2,17 @@ import prisma from '@/prisma/client' import debug from '@/lib/debug' -import { captureError, captureException } from '@/lib/error' +import { captureException } from '@/lib/error' import { clearFastTranslationMap, getFastTranslationMap } from '@/lib/intl' import { withPost } from '@/lib/method' import { requiredUrlParam } from '@/lib/query.get' -import { notAuthorized, notFound, ok, respondFromError } from '@/lib/response' +import { + captureUnknownError, + notAuthorized, + notFound, + ok, + respondFromError, +} from '@/lib/response' import { withSession } from '@/lib/session.handler' /** @@ -144,7 +150,9 @@ export default withPost( try { await doSetup(widgetIntegration) } catch (e) { - await captureError(e) + // @note a setup refused for a missing or rejected configuration is a + // conflict the caller reads, not a fault + await captureUnknownError(e) return respondFromError(e) } diff --git a/platform/pages/api/v1/skillset/[skillsetId]/ability/[abilityId]/_execute.utest.js b/platform/pages/api/v1/skillset/[skillsetId]/ability/[abilityId]/_execute.utest.js index 25dc7c0..5c100c7 100644 --- a/platform/pages/api/v1/skillset/[skillsetId]/ability/[abilityId]/_execute.utest.js +++ b/platform/pages/api/v1/skillset/[skillsetId]/ability/[abilityId]/_execute.utest.js @@ -47,6 +47,8 @@ jest.mock('@/lib/usage.model', () => ({ })) jest.mock('@/lib/response', () => ({ + // @note the real filter, so a known-code error is not captured + captureUnknownError: jest.requireActual('@/lib/response').captureUnknownError, throwNotFound: jest.fn(() => { throw Object.assign(new Error('Not found'), { code: 'NOT_FOUND' }) }), diff --git a/platform/pages/api/v1/skillset/[skillsetId]/ability/[abilityId]/execute.js b/platform/pages/api/v1/skillset/[skillsetId]/ability/[abilityId]/execute.js index 5d61ab4..f8efcd1 100644 --- a/platform/pages/api/v1/skillset/[skillsetId]/ability/[abilityId]/execute.js +++ b/platform/pages/api/v1/skillset/[skillsetId]/ability/[abilityId]/execute.js @@ -4,17 +4,17 @@ import prisma from '@/prisma/client' import { setContextContact, setContextNamespace } from '@/lib/context.store' import { TAG_ERROR, TAG_RESULT, createSinkEvent } from '@/lib/conversation.tag' import { withStream } from '@/lib/stream' -import { - captureError, - errorResponseToError, - errorToSafeErrorResponse, -} from '@/lib/error' +import { errorResponseToError, errorToSafeErrorResponse } from '@/lib/error' import schema, { withSchema } from '@/lib/joi.handler' import { withSessionLimits } from '@/lib/limit.handler' import { withPost } from '@/lib/method' import { getSafeNamespace } from '@/lib/namespace.safe' import { requiredUrlParam } from '@/lib/query.get' -import { throwNotAuthorized, throwNotFound } from '@/lib/response' +import { + captureUnknownError, + throwNotAuthorized, + throwNotFound, +} from '@/lib/response' import { applySkillset } from '@/lib/skillset.apply' import { Usage } from '@/lib/usage.model' @@ -285,7 +285,7 @@ export default withPost( messages, }) } catch (e) { - await captureError(e) + await captureUnknownError(e) const event = createSinkEvent({ type: TAG_ERROR, diff --git a/platform/schemas/dbSourceUrl.js b/platform/schemas/dbSourceUrl.js new file mode 100644 index 0000000..fb9d65c --- /dev/null +++ b/platform/schemas/dbSourceUrl.js @@ -0,0 +1,13 @@ +// @ts-check +import { MAX_DB_SOURCE_URL_BYTES_LENGTH } from '@/prisma/constraints' + +import schema from '@/lib/joi.schema' + +/** + * A source URL - a sitemap or crawl entry point - which the schema stores in + * a wider column than a plain db string. + */ +export default schema + .string() + .allow(null, '') + .maxByteLength(MAX_DB_SOURCE_URL_BYTES_LENGTH) diff --git a/platform/schemas/dbSourceUrl.utest.js b/platform/schemas/dbSourceUrl.utest.js new file mode 100644 index 0000000..e28b507 --- /dev/null +++ b/platform/schemas/dbSourceUrl.utest.js @@ -0,0 +1,35 @@ +import { MAX_DB_SOURCE_URL_BYTES_LENGTH } from '@/prisma/constraints' + +import dbSourceUrlSchema from '@/schemas/dbSourceUrl' + +const itIfLengthIsConstrained = + MAX_DB_SOURCE_URL_BYTES_LENGTH <= 1000000 ? it : it.skip + +describe('dbSourceUrlSchema', () => { + it('should validate a url', () => { + const value = 'https://example.com/sitemap.xml' + + expect(dbSourceUrlSchema.validate(value)).toEqual({ value }) + }) + + it('should allow null and empty values', () => { + expect(dbSourceUrlSchema.validate(null)).toEqual({ value: null }) + expect(dbSourceUrlSchema.validate('')).toEqual({ value: '' }) + }) + + it('should accept a url longer than a plain db string', () => { + const value = `https://example.com/${'a'.repeat(300)}` + + expect(dbSourceUrlSchema.validate(value)).toEqual({ value }) + }) + + itIfLengthIsConstrained('should reject a url over the column width', () => { + const value = `https://example.com/${'a'.repeat( + MAX_DB_SOURCE_URL_BYTES_LENGTH + )}` + + const { error } = dbSourceUrlSchema.validate(value) + + expect(error.message).toContain('bytes long') + }) +}) diff --git a/platform/schemas/messages.js b/platform/schemas/messages.js index 29a6e4f..bedc216 100644 --- a/platform/schemas/messages.js +++ b/platform/schemas/messages.js @@ -8,59 +8,75 @@ import messageText from '@/schemas/messageText' import metaSchema from '@/schemas/meta' import nameSchema from '@/schemas/name' -export const messageSchema = schema - .object({ - name: nameSchema, - description: descriptionSchema, - - type: schema - .string() - .valid(...Object.keys(MessageType)) - .required(), - text: messageText.required(), - - meta: metaSchema, - - // @note these are not used but required to simplify some development workflows - // @todo remove these fields but test before removing them - ...{ - id: schema.string().allow(null, ''), - createdAt: schema.any().allow(null), - }, - }) - .external((message) => { - if (message.type === 'activity') { - const { meta } = message - - if (!meta) { - throw new Error(`missing 'meta' for message of type 'activity'`) - } - - const { activity } = meta - - if (!activity) { - throw new Error( - `missing 'activity' in 'meta' for message of type 'activity'` - ) - } - - const { type: _type, function: _function } = activity - - if (!['request', 'response', 'trigger'].includes(_type)) { - throw new Error( - `invalid 'meta.activity.type' for message of type 'activity'` - ) - } - - if (!_function) { - throw new Error( - `missing 'meta.activity.function' for message of type 'activity'` - ) - } +/** + * An activity message is a function trigger, request or response and the + * model side of the conversation relies on its meta to say which - see + * `lib/message.ts`. Throws when the meta does not describe one. + * + * @param {{ type?: string, meta?: any }} message + * @throws {Error} when the message is of type 'activity' but its meta does not describe a valid activity + */ +export function assertActivityMessage(message) { + if (message.type === 'activity') { + const { meta } = message + + if (!meta) { + throw new Error(`missing 'meta' for message of type 'activity'`) } - return message - }) + const { activity } = meta + + if (!activity) { + throw new Error( + `missing 'activity' in 'meta' for message of type 'activity'` + ) + } + + const { type: _type, function: _function } = activity + + if (!['request', 'response', 'trigger'].includes(_type)) { + throw new Error( + `invalid 'meta.activity.type' for message of type 'activity'` + ) + } + + if (!_function) { + throw new Error( + `missing 'meta.activity.function' for message of type 'activity'` + ) + } + } +} + +/** + * The fields of a message without the activity rule - for a route that takes + * a partial message and has to check the rule against what is stored. + */ +export const messageFieldsSchema = schema.object({ + name: nameSchema, + description: descriptionSchema, + + type: schema + .string() + .valid(...Object.keys(MessageType)) + .required(), + text: messageText.required(), + + meta: metaSchema, + + // @note these are not used but required to simplify some development workflows + // @todo remove these fields but test before removing them + ...{ + id: schema.string().allow(null, ''), + createdAt: schema.any().allow(null), + }, +}) + +export const messageSchema = messageFieldsSchema.external((message) => { + assertActivityMessage(message) + + return message +}) export const allMessagesSchema = schema.array().items(messageSchema).allow(null)