diff --git a/CHANGELOG.md b/CHANGELOG.md index 53f046eba6..559f2201f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,31 @@ --- +## Unreleased + +- ✨ Two new init options keep only the sessions that report an error, for customers who want every + error investigated without storing and paying for every session. `sessionOnError` keeps the + events of a session the plain `sessionSampleRate` draw missed: it records from the start, uploads + nothing, and is never stored unless it reports an error — on the first error the withheld history, + up to the last minute of it, is uploaded and collection continues. `sessionReplayOnError` does the + same for the Session Replay of a session the plain `sessionReplaySampleRate` draw missed. Both are + switches, default off, and apply only to what the plain rate did not already draw, so a session is + never counted twice. Both can also be set from the console when `remoteConfigurationEnabled` is on. + View events of such a session carry `sampled_for_error` / `sampled_for_error_replay` so a stored + error session can be told apart from an ordinary one. + + Known limitations of the on-error switches: + + - A site that gates recording on consent by calling `startSessionReplayRecording()` itself must set + `startSessionReplayRecordingManually: true` explicitly. With `remoteConfigurationEnabled` on and an + init replay rate of 0, the recorder now starts on its own so a console-delivered rate has something + to withhold — which would otherwise begin recording before the consent call. + - On a single-page app, the released replay reaches back only to the start of the view the error + happened in, while the released events reach back the full minute across views. + - With the opt-in `compressIntakeRequests`, closing the tab within a few seconds of a session's first + error can lose that release: the burst is then too large for `sendBeacon` and the exit fetch is + cancelled by the unload. The default (uncompressed) path is not affected. + ## v0.2.3 - ✨ A session sample rate published from the console that rises above 0 now ends the running diff --git a/packages/core/src/domain/session/sessionManager.spec.ts b/packages/core/src/domain/session/sessionManager.spec.ts index d709e3b54f..2cfd2d39c9 100644 --- a/packages/core/src/domain/session/sessionManager.spec.ts +++ b/packages/core/src/domain/session/sessionManager.spec.ts @@ -665,7 +665,7 @@ describe('startSessionManager', () => { const sessionManager = startSessionManagerWithDefaults() sessionManager.sessionStateUpdateObservable.subscribe(sessionStateUpdateSpy) - sessionManager.updateSessionState({ extra: 'extra' }) + sessionManager.updateSessionState(() => ({ extra: 'extra' })) expectSessionIdToBeDefined(sessionManager) expect(sessionStateUpdateSpy).toHaveBeenCalledTimes(1) diff --git a/packages/core/src/domain/session/sessionManager.ts b/packages/core/src/domain/session/sessionManager.ts index 03caf2f49c..fae90339a5 100644 --- a/packages/core/src/domain/session/sessionManager.ts +++ b/packages/core/src/domain/session/sessionManager.ts @@ -20,13 +20,19 @@ export interface SessionManager { expireObservable: Observable sessionStateUpdateObservable: Observable<{ previousState: SessionState; newState: SessionState }> expire: () => void - updateSessionState: (state: Partial) => void + updateSessionState: (update: (state: SessionState) => Partial | undefined) => void } export interface SessionContext extends Context { id: string trackingType: TrackingType isReplayForced: boolean + /** + * Whether an error has already been reported during this session. Persisted in the session store + * so it survives page navigation: an error session must not go back to withholding its replay + * just because the user moved to another page. + */ + hasError: boolean anonymousId: string | undefined } @@ -92,6 +98,7 @@ export function startSessionManager( id: sessionStore.getSession().id!, trackingType: sessionStore.getSession()[productKey] as TrackingType, isReplayForced: !!sessionStore.getSession().forcedReplay, + hasError: !!sessionStore.getSession().hasError, anonymousId: sessionStore.getSession().anonymousId, } } diff --git a/packages/core/src/domain/session/sessionStore.spec.ts b/packages/core/src/domain/session/sessionStore.spec.ts index 7d8105177c..2bcd9378f7 100644 --- a/packages/core/src/domain/session/sessionStore.spec.ts +++ b/packages/core/src/domain/session/sessionStore.spec.ts @@ -596,7 +596,7 @@ describe('session store', () => { sessionStoreManager = setupSessionStore(updateSpy) otherSessionStoreManager = setupSessionStore(otherUpdateSpy) - sessionStoreManager.updateSessionState({ extra: 'extra' }) + sessionStoreManager.updateSessionState(() => ({ extra: 'extra' })) expect(updateSpy).toHaveBeenCalledTimes(1) diff --git a/packages/core/src/domain/session/sessionStore.ts b/packages/core/src/domain/session/sessionStore.ts index c926e79410..cb2253a9e6 100644 --- a/packages/core/src/domain/session/sessionStore.ts +++ b/packages/core/src/domain/session/sessionStore.ts @@ -29,7 +29,12 @@ export interface SessionStore { sessionStateUpdateObservable: Observable<{ previousState: SessionState; newState: SessionState }> expire: () => void stop: () => void - updateSessionState: (state: Partial) => void + /** + * Applies a change to the stored session under the store lock. The producer sees the state the + * change would land on and returns `undefined` to make it a no-op - which is how a write meant for + * one session avoids landing on the one that replaced it while the write was waiting for the lock. + */ + updateSessionState: (update: (state: SessionState) => Partial | undefined) => void } /** @@ -216,10 +221,13 @@ export function startSessionStore( renewObservable.notify() } - function updateSessionState(partialSessionState: Partial) { + function updateSessionState(update: (state: SessionState) => Partial | undefined) { processSessionStoreOperations( { - process: (sessionState) => ({ ...sessionState, ...partialSessionState }), + process: (sessionState) => { + const partialSessionState = update(sessionState) + return partialSessionState && { ...sessionState, ...partialSessionState } + }, after: synchronizeSession, }, sessionStoreStrategy diff --git a/packages/core/src/domain/session/sessionStoreOperations.spec.ts b/packages/core/src/domain/session/sessionStoreOperations.spec.ts index 78d76b6efe..37a8ab374f 100644 --- a/packages/core/src/domain/session/sessionStoreOperations.spec.ts +++ b/packages/core/src/domain/session/sessionStoreOperations.spec.ts @@ -1,3 +1,4 @@ +import { startFakeTelemetry } from '../telemetry' import type { MockStorage } from '../../../test' import { mockClock, mockCookie, mockLocalStorage } from '../../../test' import type { CookieOptions } from '../../browser/cookie' @@ -232,6 +233,7 @@ const DEFAULT_INIT_CONFIGURATION = { trackAnonymousUser: true } as Configuration it('should abort after a max number of retry', () => { const clock = mockClock() + const telemetry = startFakeTelemetry() sessionStoreStrategy.persistSession(initialSession) storage.setSpy.calls.reset() @@ -246,6 +248,7 @@ const DEFAULT_INIT_CONFIGURATION = { trackAnonymousUser: true } as Configuration expect(processSpy).not.toHaveBeenCalled() expect(afterSpy).not.toHaveBeenCalled() expect(storage.setSpy).not.toHaveBeenCalled() + expect(telemetry).toContain(jasmine.objectContaining({ message: 'Session store lock retries exhausted' })) clock.cleanup() }) diff --git a/packages/core/src/domain/session/sessionStoreOperations.ts b/packages/core/src/domain/session/sessionStoreOperations.ts index 869347d0cc..5be4d4cc5f 100644 --- a/packages/core/src/domain/session/sessionStoreOperations.ts +++ b/packages/core/src/domain/session/sessionStoreOperations.ts @@ -1,3 +1,4 @@ +import { addTelemetryDebug } from '../telemetry' import { setTimeout } from '../../tools/timer' import { generateUUID } from '../../tools/utils/stringUtils' import type { SessionStoreStrategy } from './storeStrategies/sessionStoreStrategy' @@ -43,6 +44,7 @@ export function processSessionStoreOperations( return } if (isLockEnabled && numberOfRetries >= LOCK_MAX_TRIES) { + addTelemetryDebug('Session store lock retries exhausted', { retries: numberOfRetries }) next(sessionStoreStrategy) return } diff --git a/packages/core/src/domain/telemetry/telemetry.ts b/packages/core/src/domain/telemetry/telemetry.ts index 4b0a12c617..4651ebb253 100644 --- a/packages/core/src/domain/telemetry/telemetry.ts +++ b/packages/core/src/domain/telemetry/telemetry.ts @@ -4,7 +4,9 @@ import { NO_ERROR_STACK_PRESENT_MESSAGE, isError } from '../error/error' import { toStackTraceString } from '../../tools/stackTrace/handlingStack' import { getExperimentalFeatures } from '../../tools/experimentalFeatures' import type { Configuration } from '../configuration' -import { INTAKE_SITE_STAGING } from '../configuration' +// Import the constant without loading configuration construction, which uses session storage. +// eslint-disable-next-line local-rules/disallow-protected-directory-import +import { INTAKE_SITE_STAGING } from '../configuration/intakeSites' import { Observable } from '../../tools/observable' import { timeStampNow } from '../../tools/utils/timeUtils' import { displayIfDebugEnabled, startMonitorErrorCollection } from '../../tools/monitor' diff --git a/packages/core/src/transport/flushController.ts b/packages/core/src/transport/flushController.ts index d13c672bdc..2d203db939 100644 --- a/packages/core/src/transport/flushController.ts +++ b/packages/core/src/transport/flushController.ts @@ -78,6 +78,7 @@ export function createFlushController({ } return { + flush, flushObservable, get messagesCount() { return currentMessagesCount diff --git a/packages/core/src/transport/startBatchWithReplica.ts b/packages/core/src/transport/startBatchWithReplica.ts index 1009b2dbea..1b0a3ce855 100644 --- a/packages/core/src/transport/startBatchWithReplica.ts +++ b/packages/core/src/transport/startBatchWithReplica.ts @@ -6,6 +6,7 @@ import type { RawError } from '../domain/error/error.types' import type { Encoder } from '../tools/encoder' import { createBatch } from './batch' import { createHttpRequest } from './httpRequest' +import type { FlushReason } from './flushController' import { createFlushController } from './flushController' export interface BatchConfiguration { @@ -45,6 +46,10 @@ export function startBatchWithReplica( } return { + flush: (reason: FlushReason) => { + primaryBatch.flushController.flush(reason) + replicaBatch?.flushController.flush(reason) + }, flushObservable: primaryBatch.flushController.flushObservable, add(message: T, replicated = true) { diff --git a/packages/core/test/emulate/mockFlushController.ts b/packages/core/test/emulate/mockFlushController.ts index c894908d47..60beb27d9e 100644 --- a/packages/core/test/emulate/mockFlushController.ts +++ b/packages/core/test/emulate/mockFlushController.ts @@ -8,7 +8,7 @@ export function createMockFlushController() { let currentMessagesCount = 0 let currentBytesCount = 0 - return { + const controller = { notifyBeforeAddMessage: jasmine .createSpy() .and.callFake((messageBytesCount) => { @@ -33,6 +33,11 @@ export function createMockFlushController() { return currentBytesCount }, flushObservable, + flush(reason: FlushReason) { + if (currentMessagesCount > 0) { + controller.notifyFlush(reason) + } + }, notifyFlush(reason: FlushReason = 'bytes_limit') { if (currentMessagesCount === 0) { throw new Error( @@ -53,4 +58,5 @@ export function createMockFlushController() { }) }, } satisfies Record & FlushController + return controller } diff --git a/packages/rum-core/src/boot/startRum.ts b/packages/rum-core/src/boot/startRum.ts index 06299236f4..11cb19a730 100644 --- a/packages/rum-core/src/boot/startRum.ts +++ b/packages/rum-core/src/boot/startRum.ts @@ -29,6 +29,7 @@ import { startErrorCollection } from '../domain/error/errorCollection' import { startResourceCollection } from '../domain/resource/resourceCollection' import { startViewCollection } from '../domain/view/viewCollection' import { startRumSessionManager, startRumSessionManagerStub } from '../domain/rumSessionManager' +import { startSessionErrorTracking } from '../domain/trackSessionError' import { startRumBatch } from '../transport/startRumBatch' import { startRumEventBridge } from '../transport/startRumEventBridge' import { startUrlContexts } from '../domain/contexts/urlContexts' @@ -103,6 +104,11 @@ export function startRum( } const pageMayExitObservable = createPageMayExitObservable(configuration) + // Subscribed before the batch below, and it has to stay that way. The batch flushes on this same + // observable, and observers run in the order they subscribed - so the withheld event buffer, which + // releases on the lifecycle notification raised here, has to get its events into the batch before + // the flush that is the page's last chance to send them. The same holds for the session expiry + // relay in `startRumSessionManager`, which the session manager registers just below. const pageMayExitSubscription = pageMayExitObservable.subscribe((event) => { lifeCycle.notify(LifeCycleEventType.PAGE_MAY_EXIT, event) }) @@ -121,6 +127,12 @@ export function startRum( : startRumSessionManager(configuration, lifeCycle, trackingConsentState) cleanupTasks.push(session.stop) + // Subscribed before the batch below, and it has to stay that way: the withheld event buffer runs + // on the same event, and only sees a session as released if this has already marked it. Reorder + // them and the release waits for whatever event happens to come next. + const sessionErrorTracking = startSessionErrorTracking(lifeCycle, session) + cleanupTasks.push(() => sessionErrorTracking.stop()) + if (!canUseEventBridge()) { // FLASHCAT FORK - keep the console's sampling rates fresh, at the rhythm the sessions read // them: once now and once per session renewal. It is skipped under an event bridge, where the @@ -135,7 +147,7 @@ export function startRum( telemetry.observable, reportError, pageMayExitObservable, - session.expireObservable, + session, createEncoder ) cleanupTasks.push(() => batch.stop()) diff --git a/packages/rum-core/src/domain/configuration/configuration.spec.ts b/packages/rum-core/src/domain/configuration/configuration.spec.ts index 64069cd281..8dc6853653 100644 --- a/packages/rum-core/src/domain/configuration/configuration.spec.ts +++ b/packages/rum-core/src/domain/configuration/configuration.spec.ts @@ -65,6 +65,184 @@ describe('validateAndBuildRumConfiguration', () => { }) }) + describe('sessionReplayOnError', () => { + it('is carried into the built configuration', () => { + expect( + validateAndBuildRumConfiguration({ ...DEFAULT_INIT_CONFIGURATION, sessionReplayOnError: true })! + .sessionReplayOnError + ).toBeTrue() + }) + + it('defaults to collecting no error replay at all', () => { + expect(validateAndBuildRumConfiguration(DEFAULT_INIT_CONFIGURATION)!.sessionReplayOnError).toBeFalse() + }) + + it('is read as a switch, whatever it was given', () => { + expect( + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionReplayOnError: 1 as unknown as boolean, + })!.sessionReplayOnError + ).toBeTrue() + }) + + it('starts the recording on its own, since there is nothing to withhold otherwise', () => { + expect( + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionReplaySampleRate: 0, + sessionReplayOnError: true, + })!.startSessionReplayRecordingManually + ).toBeFalse() + }) + + it('warns when the plain replay rate leaves it nothing to apply to', () => { + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionReplaySampleRate: 100, + sessionReplayOnError: true, + }) + + expect(displayWarnSpy).toHaveBeenCalledTimes(1) + expect(displayWarnSpy.calls.argsFor(0)[0]).toContain('sessionReplaySampleRate did not draw') + }) + + it('warns when no session is tracked at all', () => { + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionSampleRate: 0, + sessionReplayOnError: true, + }) + + expect(displayWarnSpy).toHaveBeenCalledTimes(1) + expect(displayWarnSpy.calls.argsFor(0)[0]).toContain('no session is tracked') + }) + + it('warns when the recording is left for the customer to start, since nothing would be held', () => { + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionReplayOnError: true, + startSessionReplayRecordingManually: true, + }) + + expect(displayWarnSpy).toHaveBeenCalledTimes(1) + expect(displayWarnSpy.calls.argsFor(0)[0]).toContain('startSessionReplayRecordingManually') + }) + + it('says nothing about a switch that can apply', () => { + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionReplaySampleRate: 20, + sessionReplayOnError: true, + }) + + expect(displayWarnSpy).not.toHaveBeenCalled() + }) + }) + + describe('sessionOnError', () => { + it('is carried into the built configuration', () => { + expect( + validateAndBuildRumConfiguration({ ...DEFAULT_INIT_CONFIGURATION, sessionOnError: true })!.sessionOnError + ).toBeTrue() + }) + + it('defaults to collecting no error-only session at all', () => { + expect(validateAndBuildRumConfiguration(DEFAULT_INIT_CONFIGURATION)!.sessionOnError).toBeFalse() + }) + + it('is read as a switch, whatever it was given', () => { + expect( + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionOnError: 'yes' as unknown as boolean, + })!.sessionOnError + ).toBeTrue() + }) + + it('warns when the replay it would withhold is never recorded', () => { + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionSampleRate: 20, + sessionOnError: true, + sessionReplaySampleRate: 50, + startSessionReplayRecordingManually: true, + }) + + expect(displayWarnSpy).toHaveBeenCalledTimes(1) + expect(displayWarnSpy.calls.argsFor(0)[0]).toContain('startSessionReplayRecordingManually') + }) + + it('says nothing about a replay it could never withhold anyway', () => { + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionOnError: true, + sessionReplaySampleRate: 30, + startSessionReplayRecordingManually: true, + }) + + // the switch cannot apply at all here, which is the one thing worth saying + expect(displayWarnSpy).toHaveBeenCalledTimes(1) + expect(displayWarnSpy.calls.argsFor(0)[0]).toContain('sessionSampleRate did not draw') + }) + + it('does not warn about manual recording when replay is disabled for the on-error session', () => { + // there is nothing to withhold on the replay side, so the manual-start warning does not apply - + // even though the plain session rate leaves room for the switch and recording is manual + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionSampleRate: 20, + sessionOnError: true, + sessionReplaySampleRate: 0, + startSessionReplayRecordingManually: true, + }) + + expect(displayWarnSpy).not.toHaveBeenCalled() + }) + + it('warns when the default session rate leaves it nothing to apply to', () => { + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionOnError: true, + }) + + expect(displayWarnSpy).toHaveBeenCalledTimes(1) + }) + + it('stays silent under remote configuration, where the console owns the session rate', () => { + // the documented remote-config setup: the site omits the rate and lets the console deliver it, + // so the init default of 100 is a fallback, not the rate the switch will face + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionOnError: true, + remoteConfigurationEnabled: true, + }) + + expect(displayWarnSpy).not.toHaveBeenCalled() + }) + + it('says nothing once the plain session rate leaves room for it', () => { + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionSampleRate: 20, + sessionOnError: true, + }) + + expect(displayWarnSpy).not.toHaveBeenCalled() + }) + + it('makes the replay-on-error switch meaningful even with no plainly sampled session', () => { + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionSampleRate: 0, + sessionOnError: true, + sessionReplayOnError: true, + }) + + expect(displayWarnSpy).not.toHaveBeenCalled() + }) + }) + describe('traceSampleRate', () => { it('defaults to 100 if the option is not provided', () => { expect(validateAndBuildRumConfiguration(DEFAULT_INIT_CONFIGURATION)!.traceSampleRate).toBe(100) @@ -283,6 +461,26 @@ describe('validateAndBuildRumConfiguration', () => { }) describe('startSessionReplayRecordingManually', () => { + it('keeps automatic recording available for remotely enabled replay', () => { + expect( + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + sessionReplaySampleRate: 0, + remoteConfigurationEnabled: true, + })!.startSessionReplayRecordingManually + ).toBeFalse() + }) + + it('respects explicit manual recording when remote configuration is enabled', () => { + expect( + validateAndBuildRumConfiguration({ + ...DEFAULT_INIT_CONFIGURATION, + remoteConfigurationEnabled: true, + startSessionReplayRecordingManually: true, + })!.startSessionReplayRecordingManually + ).toBeTrue() + }) + it('defaults to true if sessionReplaySampleRate is 0', () => { expect( validateAndBuildRumConfiguration({ ...DEFAULT_INIT_CONFIGURATION, sessionReplaySampleRate: 0 })! @@ -554,6 +752,8 @@ describe('serializeRumConfiguration', () => { enablePrivacyForActionName: false, subdomain: 'foo', sessionReplaySampleRate: 60, + sessionReplayOnError: true, + sessionOnError: true, startSessionReplayRecordingManually: true, sessionReplayDirectUpload: true, trackUserInteractions: true, @@ -587,6 +787,9 @@ describe('serializeRumConfiguration', () => { // FLASHCAT FORK: not reported to telemetry | 'sessionReplayDirectUpload' | 'beforeSampling' + // not reported yet: needs a rum-events-format schema change first + | 'sessionReplayOnError' + | 'sessionOnError' ? never : CamelToSnakeCase // By specifying the type here, we can ensure that serializeConfiguration is returning an diff --git a/packages/rum-core/src/domain/configuration/configuration.ts b/packages/rum-core/src/domain/configuration/configuration.ts index f449070d4f..35e873fb49 100644 --- a/packages/rum-core/src/domain/configuration/configuration.ts +++ b/packages/rum-core/src/domain/configuration/configuration.ts @@ -209,7 +209,39 @@ export interface RumInitConfiguration extends InitConfiguration { */ sessionReplaySampleRate?: number | undefined /** - * If the session is sampled for Session Replay, only start the recording when `startSessionReplayRecording()` is called, instead of at the beginning of the session. Default: if startSessionReplayRecording is 0, true; otherwise, false. + * Whether the tracked sessions that `sessionReplaySampleRate` did not draw still record a replay, + * uploaded only if the session reports an error. Default: false. + * + * Such a session records from the start and keeps at most the last minute of it in memory. If it + * never reports an error, nothing is uploaded and the session is not billed. On the first error, + * the withheld minute is uploaded and recording continues normally for the rest of the session. + * + * The withheld replay does not span a view change: what is released reaches back to the start of + * the view the error happened in, not a full minute across earlier views. The session's events + * (see `sessionOnError`) do reach back the full minute across views. + */ + sessionReplayOnError?: boolean | undefined + /** + * Whether the sessions that `sessionSampleRate` did not draw still collect events, uploaded only + * if the session reports an error. Default: false. It only applies to what the plain rate missed, + * so with the default `sessionSampleRate` of 100 there is nothing left for it to apply to. + * + * Such a session collects from the start and keeps at most the last minute of it in memory. If it + * never reports an error, nothing is uploaded and the session is not stored. On the first error, + * the withheld minute is uploaded and collection continues normally. + * + * A session kept this way never uploads its replay ahead of its events: until the events are + * released the session does not exist yet, and a replay sent then would have nothing to attach to. + */ + sessionOnError?: boolean | undefined + /** + * If the session is sampled for Session Replay, only start the recording when `startSessionReplayRecording()` is called, instead of at the beginning of the session. + * + * Default when left unset: `true` only if `sessionReplaySampleRate` is 0, `sessionReplayOnError` is + * off, and `remoteConfigurationEnabled` is not set; `false` otherwise. A session kept by + * `sessionReplayOnError`, or one whose replay rate may be raised from the console, has to be + * recording before the error happens, so the recording must start on its own rather than wait for a + * manual call. * See [Session Replay Usage](https://docs.datadoghq.com/real_user_monitoring/session_replay/browser/#usage) for further information. */ startSessionReplayRecordingManually?: boolean | undefined @@ -309,6 +341,8 @@ export interface RumConfiguration extends Configuration { defaultPrivacyLevel: DefaultPrivacyLevel enablePrivacyForActionName: boolean sessionReplaySampleRate: number + sessionReplayOnError: boolean + sessionOnError: boolean startSessionReplayRecordingManually: boolean sessionReplayDirectUpload: boolean trackUserInteractions: boolean @@ -393,16 +427,65 @@ export function validateAndBuildRumConfiguration( const profilingEnabled = isExperimentalFeatureEnabled(ExperimentalFeature.PROFILING) const sessionReplaySampleRate = initConfiguration.sessionReplaySampleRate ?? 0 + const sessionReplayOnError = !!initConfiguration.sessionReplayOnError + const sessionOnError = !!initConfiguration.sessionOnError + + // Each of the cases below is a combination the customer can set that cannot apply to a single + // session. It is valid, so validation lets it through - but silence would leave someone waiting + // for data that is never coming. + // + // Only judged against the init rates when the console cannot change them: under remote + // configuration these values are a fallback until the first fetch lands, so the console may + // deliver the very rate that leaves the switch room to apply. Warning on the init values there + // would fire on the documented remote-config setup - a site that omits the rate and lets the + // console own it - which is exactly not a misconfiguration. + if (!initConfiguration.remoteConfigurationEnabled) { + if (sessionOnError && (initConfiguration.sessionSampleRate ?? 100) === 100) { + display.warn( + 'sessionOnError only applies to sessions sessionSampleRate did not draw, and that rate is 100: it will never apply.' + ) + } + if (sessionReplayOnError) { + if (sessionReplaySampleRate === 100) { + display.warn( + 'sessionReplayOnError only applies to sessions sessionReplaySampleRate did not draw, and that rate is 100: it will never apply.' + ) + } + if ((initConfiguration.sessionSampleRate ?? 100) === 0 && !sessionOnError) { + display.warn( + 'sessionReplayOnError has no effect while sessionSampleRate is 0 and sessionOnError is off: no session is tracked.' + ) + } + } + } + + // A session kept on error withholds whichever replay it draws, so the same trap is reachable + // through the plain replay rate as well - and there it is worse than silence, since the released + // views would report a replay for a recording that never ran. + if ( + initConfiguration.startSessionReplayRecordingManually && + (sessionReplayOnError || + (sessionOnError && sessionReplaySampleRate > 0 && (initConfiguration.sessionSampleRate ?? 100) < 100)) + ) { + display.warn( + 'A replay kept until the session errors has to be recording before that error, and startSessionReplayRecordingManually keeps it stopped until you start it: there would be nothing to release.' + ) + } return { applicationId: initConfiguration.applicationId, version: initConfiguration.version || undefined, actionNameAttribute: initConfiguration.actionNameAttribute, sessionReplaySampleRate, + sessionReplayOnError, + sessionOnError, startSessionReplayRecordingManually: initConfiguration.startSessionReplayRecordingManually !== undefined ? !!initConfiguration.startSessionReplayRecordingManually - : sessionReplaySampleRate === 0, + : // An error-sampled session has to be recording before the error happens, otherwise there is + // nothing to withhold and release. Remote configuration may enable replay on a later + // session, so keep the automatic start intent even when init disables replay. + sessionReplaySampleRate === 0 && !sessionReplayOnError && !initConfiguration.remoteConfigurationEnabled, sessionReplayDirectUpload: !!initConfiguration.sessionReplayDirectUpload, traceSampleRate: initConfiguration.traceSampleRate ?? 100, rulePsr: isNumber(initConfiguration.traceSampleRate) ? initConfiguration.traceSampleRate / 100 : undefined, @@ -493,6 +576,9 @@ export function serializeRumConfiguration(configuration: RumInitConfiguration) { return { session_replay_sample_rate: configuration.sessionReplaySampleRate, + // `session_replay_on_error` and `session_on_error` are deliberately not reported yet: the telemetry + // configuration type is generated from the rum-events-format schema, so adding it needs a schema + // change first, and that is a separate repository. start_session_replay_recording_manually: configuration.startSessionReplayRecordingManually, trace_sample_rate: configuration.traceSampleRate, trace_context_injection: configuration.traceContextInjection, diff --git a/packages/rum-core/src/domain/configuration/remoteConfiguration.spec.ts b/packages/rum-core/src/domain/configuration/remoteConfiguration.spec.ts index c3e409ab8c..df490e069a 100644 --- a/packages/rum-core/src/domain/configuration/remoteConfiguration.spec.ts +++ b/packages/rum-core/src/domain/configuration/remoteConfiguration.spec.ts @@ -121,6 +121,34 @@ describe('remoteConfiguration', () => { start(configurationWith()) }) + it('keeps the on-error switches the server reports, either way they are set', (done) => { + interceptor.withMockXhr((xhr) => { + xhr.complete( + 200, + body({ rum: { sessionReplaySampleRate: 10, sessionReplayOnError: false, sessionOnError: true } }) + ) + + expect(readRemoteConfig(setup)).toEqual({ + sessionReplaySampleRate: 10, + sessionReplayOnError: false, + sessionOnError: true, + version: 3, + }) + done() + }) + start(configurationWith()) + }) + + it('drops a switch that is not a boolean, so it reads as not delivered', (done) => { + interceptor.withMockXhr((xhr) => { + xhr.complete(200, body({ rum: { sessionSampleRate: 50, sessionReplayOnError: 'true' as unknown as boolean } })) + + expect(readRemoteConfig(setup)).toEqual({ sessionSampleRate: 50, version: 3 }) + done() + }) + start(configurationWith()) + }) + it('drops a privacy level it does not recognise rather than passing it on', (done) => { // A typo must not reach the recorders: an unknown value there falls through to recording // everything, which is the one outcome nobody asks for by accident. diff --git a/packages/rum-core/src/domain/configuration/remoteConfiguration.ts b/packages/rum-core/src/domain/configuration/remoteConfiguration.ts index 2c0ee781f9..84b3cc34a1 100644 --- a/packages/rum-core/src/domain/configuration/remoteConfiguration.ts +++ b/packages/rum-core/src/domain/configuration/remoteConfiguration.ts @@ -102,6 +102,17 @@ export interface RemoteConfigValues { * fact. */ defaultPrivacyLevel?: DefaultPrivacyLevel + /** + * Whether the sessions `sessionReplaySampleRate` did not draw still record a replay, uploaded only + * if the session errors. Read at the draw like the rates, and for the same reason: a session + * either withholds its replay from the start or never does. + */ + sessionReplayOnError?: boolean + /** + * Whether the sessions `sessionSampleRate` did not draw still collect, uploaded only if the + * session errors. Same footing as the replay switch above. + */ + sessionOnError?: boolean /** * Which version of the settings these rates came from. Reported back on the next request so the * console can say how far a change has actually reached — a question the events cannot answer, @@ -261,6 +272,12 @@ function readStoredValues(parsed: unknown): RemoteConfigValues { if (isPrivacyLevel(stored.defaultPrivacyLevel)) { values.defaultPrivacyLevel = stored.defaultPrivacyLevel } + if (isSwitch(stored.sessionReplayOnError)) { + values.sessionReplayOnError = stored.sessionReplayOnError + } + if (isSwitch(stored.sessionOnError)) { + values.sessionOnError = stored.sessionOnError + } if (isBag(stored.custom)) { values.custom = stored.custom } @@ -484,6 +501,14 @@ function store(setup: RemoteConfigSetup, response: RemoteConfigurationResponse) if (isPrivacyLevel(response.rum.defaultPrivacyLevel)) { values.defaultPrivacyLevel = response.rum.defaultPrivacyLevel } + // A switch is a boolean or nothing. Anything else - a "true" string, a 1 - is dropped for the + // same reason a bad rate is: it must read as "not delivered", not as either position. + if (isSwitch(response.rum.sessionReplayOnError)) { + values.sessionReplayOnError = response.rum.sessionReplayOnError + } + if (isSwitch(response.rum.sessionOnError)) { + values.sessionOnError = response.rum.sessionOnError + } } // The custom bag rides along untouched — the platform's job is delivery, its meaning belongs to // the host application. Gone from the response (or the kill switch off) means gone from storage. @@ -732,6 +757,10 @@ export function isRate(value: unknown): value is number { return typeof value === 'number' && value >= 0 && value <= 100 } +export function isSwitch(value: unknown): value is boolean { + return typeof value === 'boolean' +} + /** * A version is a publish counter, so anything that is not a whole, non-negative number small enough * to survive a JSON round trip cannot be one. Checked on the way in and on the way out, because a diff --git a/packages/rum-core/src/domain/contexts/sessionContext.spec.ts b/packages/rum-core/src/domain/contexts/sessionContext.spec.ts index 263bb737c5..1a2dc35263 100644 --- a/packages/rum-core/src/domain/contexts/sessionContext.spec.ts +++ b/packages/rum-core/src/domain/contexts/sessionContext.spec.ts @@ -94,6 +94,63 @@ describe('session context', () => { expect(eventWithoutHasReplay.session!.has_replay).toBeUndefined() }) + it('should tell a replay kept only because the session errored apart from an unconditional one', () => { + sessionManager.setTrackedWithErrorSessionReplay() + const errorReplayEvent = hooks.triggerHook(HookNames.Assemble, { + eventType: 'view', + startTime: 0 as RelativeTime, + }) as DefaultRumEventAttributes + + sessionManager.setTrackedWithSessionReplay() + const plainEvent = hooks.triggerHook(HookNames.Assemble, { + eventType: 'view', + startTime: 0 as RelativeTime, + }) as DefaultRumEventAttributes + + expect(errorReplayEvent.session!.sampled_for_error_replay).toBeTrue() + // absent rather than false, so it costs nothing on every ordinary session + expect(plainEvent.session!.sampled_for_error_replay).toBeUndefined() + }) + + it('does not report sampled_for_replay for an error-replay session that has not errored', () => { + // a type-3 session withholds only its replay, not its events; its events ship on their own, so + // reporting sampled_for_replay before the error would claim a replay for a recording that may + // never be sent + sessionManager.setTrackedWithErrorSessionReplay() + + const event = hooks.triggerHook(HookNames.Assemble, { + eventType: 'view', + startTime: 0 as RelativeTime, + }) as DefaultRumEventAttributes + + expect(event.session!.sampled_for_replay).toBe(false) + }) + + it('should not set hasReplay when a dropped buffer left the view with nothing', () => { + // a withheld buffer that was dropped rolls back what it held, and a view left with an empty + // stats entry has no replay to offer + getReplayStatsSpy.and.returnValue({ segments_count: 0, records_count: 0, segments_total_raw_size: 0 }) + + const event = hooks.triggerHook(HookNames.Assemble, { + eventType: 'view', + startTime: 0 as RelativeTime, + }) as DefaultRumEventAttributes + + expect(event.session!.has_replay).toBeUndefined() + }) + + it('should set hasReplay when a host bridge took the records and no segment was built', () => { + // records go straight to the bridge, so nothing ever counts a segment for them + getReplayStatsSpy.and.returnValue({ ...fakeStats, segments_count: 0, records_count: 10 }) + + const event = hooks.triggerHook(HookNames.Assemble, { + eventType: 'view', + startTime: 0 as RelativeTime, + }) as DefaultRumEventAttributes + + expect(event.session!.has_replay).toBe(true) + }) + it('should set session.is_active when the session is active', () => { findViewSpy.and.returnValue({ ...fakeView, sessionIsActive: true }) const eventWithActiveSession = hooks.triggerHook(HookNames.Assemble, { @@ -127,6 +184,70 @@ describe('session context', () => { expect(eventSampledOutForReplay.session!.sampled_for_replay).toBe(false) }) + it('should set sampled_for_replay on a session whose events are withheld alongside its replay', () => { + // these events only ever leave together with that replay, so reporting the state as it stands + // while they are held would mark the whole released burst as having none + sessionManager.setTrackedOnErrorWithSessionReplay() + + const event = hooks.triggerHook(HookNames.Assemble, { + eventType: 'view', + startTime: 0 as RelativeTime, + }) as DefaultRumEventAttributes + + expect(event.session!.sampled_for_replay).toBe(true) + }) + + it('should not claim a replay while one is withheld, whichever way it turns out', () => { + // the segment covering this event is dropped on the next view change and sent only if the error + // comes first; the event is assembled before either, so it claims nothing + sessionManager.setTrackedOnErrorWithSessionReplay() + isRecordingSpy.and.returnValue(true) + getReplayStatsSpy.and.returnValue(fakeStats) + + const errorEvent = hooks.triggerHook(HookNames.Assemble, { + eventType: 'error', + startTime: 0 as RelativeTime, + }) as DefaultRumEventAttributes + const viewEvent = hooks.triggerHook(HookNames.Assemble, { + eventType: 'view', + startTime: 0 as RelativeTime, + }) as DefaultRumEventAttributes + + expect(errorEvent.session!.has_replay).toBeUndefined() + expect(viewEvent.session!.has_replay).toBeUndefined() + // but the session was sampled for one, and that is answerable without knowing any segment's fate + expect(viewEvent.session!.sampled_for_replay).toBe(true) + }) + + it('should not claim a replay for a session that withholds its events and has none', () => { + sessionManager.setTrackedOnError() + + const event = hooks.triggerHook(HookNames.Assemble, { + eventType: 'view', + startTime: 0 as RelativeTime, + }) as DefaultRumEventAttributes + + expect(event.session!.sampled_for_replay).toBe(false) + }) + + it('should tell the backend a session was stored only because it errored', () => { + sessionManager.setTrackedOnError() + const onErrorEvent = hooks.triggerHook(HookNames.Assemble, { + eventType: 'view', + startTime: 0 as RelativeTime, + }) as DefaultRumEventAttributes + + sessionManager.setTrackedWithSessionReplay() + const plainEvent = hooks.triggerHook(HookNames.Assemble, { + eventType: 'view', + startTime: 0 as RelativeTime, + }) as DefaultRumEventAttributes + + expect(onErrorEvent.session!.sampled_for_error).toBeTrue() + // absent rather than false, so it costs nothing on every ordinary session + expect(plainEvent.session!.sampled_for_error).toBeUndefined() + }) + it('should report the configuration the session was drawn under', () => { sessionManager.setDrawnConfiguration({ version: 12, diff --git a/packages/rum-core/src/domain/contexts/sessionContext.ts b/packages/rum-core/src/domain/contexts/sessionContext.ts index ced3bebbee..30c63198f9 100644 --- a/packages/rum-core/src/domain/contexts/sessionContext.ts +++ b/packages/rum-core/src/domain/contexts/sessionContext.ts @@ -38,15 +38,54 @@ export function startSessionContext( return DISCARDED } + // A session withholding its replay is recording, but nothing has been uploaded and nothing may + // ever be. An event assembled now cannot know which of the two it will turn out to be: the + // segment covering it is dropped on the next view change and sent only if the error comes first, + // and it is assembled before either happens - the final update of a view is emitted before the + // view change that drops that view's segment. So it does not claim a replay. Whether the session + // was *sampled* for one is a different question, answerable here, and answered below. + const isReplayWithheld = session.sessionReplay === SessionReplayState.BUFFERED_ON_ERROR + let hasReplay let sampledForReplay + let sampledForError + let sampledForErrorReplay let isActive if (eventType === RumEventType.VIEW) { - hasReplay = recorderApi.getReplayStats(view.id) ? true : undefined - sampledForReplay = session.sessionReplay === SessionReplayState.SAMPLED + // Records rather than merely a stats entry: a withheld buffer that was dropped rolls back what + // it held, which leaves a view with an empty stats entry and no replay at all - and offering a + // replay that was never uploaded is worse than not offering one. Records, not segments, + // because a host bridge takes the records itself and no segment is ever built for them. + const replayStats = recorderApi.getReplayStats(view.id) + hasReplay = !isReplayWithheld && replayStats && replayStats.records_count > 0 ? true : undefined + // A session that withholds its events withholds its replay alongside them, so if these events + // are ever uploaded that replay is on its way with them. Reporting the state as it stands at + // assembly time would mark the whole released burst as a session that has no replay. + sampledForReplay = + session.sessionReplay === SessionReplayState.SAMPLED || (isReplayWithheld && session.eventsWithheld) + // Tells the backend that this session's detail only starts where the buffer reached, so the + // gap before it reads as "not collected" rather than as missing data. + sampledForError = session.sampledOnError || undefined + // Tells a replay collected only because the session errored apart from one collected + // unconditionally - the two cost differently and are answered by different questions. + sampledForErrorReplay = session.sampledOnErrorReplay || undefined isActive = view.sessionIsActive ? undefined : false } else { - hasReplay = recorderApi.isRecording() ? true : undefined + hasReplay = !isReplayWithheld && recorderApi.isRecording() ? true : undefined + } + + // These three are fork additions the generated event schema does not declare, so on the session + // object below they would only be checked against its `[k: string]: unknown` index signature - a + // typo in a name would compile and silently emit a field the backend never reads. Typing them + // here makes an excess or misspelled key fail the build instead. + const forkMarkers: { + sampled_for_replay: boolean | undefined + sampled_for_error: boolean | undefined + sampled_for_error_replay: boolean | undefined + } = { + sampled_for_replay: sampledForReplay, + sampled_for_error: sampledForError, + sampled_for_error_replay: sampledForErrorReplay, } return { @@ -55,7 +94,7 @@ export function startSessionContext( id: session.id, type: SessionType.USER, has_replay: hasReplay, - sampled_for_replay: sampledForReplay, + ...forkMarkers, is_active: isActive, }, // FLASHCAT FORK - overrides the init values reported by the default context with the rates diff --git a/packages/rum-core/src/domain/lifeCycle.ts b/packages/rum-core/src/domain/lifeCycle.ts index 78b6d9fccb..9f65158973 100644 --- a/packages/rum-core/src/domain/lifeCycle.ts +++ b/packages/rum-core/src/domain/lifeCycle.ts @@ -49,6 +49,8 @@ export const enum LifeCycleEventType { // at the end leaves upstream's numbering alone and keeps this file out of the way of the next // upstream merge. REMOTE_CONFIGURATION_STORED, + /** A local or shared session mark has released conditional collection. */ + SESSION_RELEASED, } // This is a workaround for an issue occurring when the Browser SDK is included in a TypeScript @@ -81,6 +83,7 @@ declare const LifeCycleEventTypeAsConst: { RAW_RUM_EVENT_COLLECTED: LifeCycleEventType.RAW_RUM_EVENT_COLLECTED RUM_EVENT_COLLECTED: LifeCycleEventType.RUM_EVENT_COLLECTED RAW_ERROR_COLLECTED: LifeCycleEventType.RAW_ERROR_COLLECTED + SESSION_RELEASED: LifeCycleEventType.SESSION_RELEASED REMOTE_CONFIGURATION_STORED: LifeCycleEventType.REMOTE_CONFIGURATION_STORED } @@ -106,6 +109,7 @@ export interface LifeCycleEventMap { error: RawError customerContext?: Context } + [LifeCycleEventTypeAsConst.SESSION_RELEASED]: { sessionId: string; reason: 'error' | 'force' } [LifeCycleEventTypeAsConst.REMOTE_CONFIGURATION_STORED]: void } diff --git a/packages/rum-core/src/domain/rumSessionManager.spec.ts b/packages/rum-core/src/domain/rumSessionManager.spec.ts index 8cf789409e..cd4b02f556 100644 --- a/packages/rum-core/src/domain/rumSessionManager.spec.ts +++ b/packages/rum-core/src/domain/rumSessionManager.spec.ts @@ -6,6 +6,7 @@ import { setCookie, stopSessionManager, ONE_SECOND, + isChromium, DOM_EVENT, createTrackingConsentState, TrackingConsent, @@ -228,6 +229,8 @@ describe('rum session manager', () => { sessionReplaySampleRate?: number traceSampleRate?: number defaultPrivacyLevel?: string + sessionReplayOnError?: boolean + sessionOnError?: boolean }) { localStorage.setItem(STORE_KEY, JSON.stringify(values)) registerCleanupTask(() => localStorage.removeItem(STORE_KEY)) @@ -255,6 +258,74 @@ describe('rum session manager', () => { expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe(RumTrackingType.TRACKED_WITH_SESSION_REPLAY) }) + it('keeps a replay on error when the console says so, over what init said', () => { + storeRemoteConfigValues({ sessionReplaySampleRate: 0, sessionReplayOnError: true }) + + startRumSessionManagerWithDefaults({ + configuration: { + sessionSampleRate: 100, + sessionReplaySampleRate: 100, + sessionReplayOnError: false, + remoteConfig: REMOTE_SAMPLING_SETUP, + }, + }) + document.dispatchEvent(createNewEvent(DOM_EVENT.CLICK)) + + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe( + RumTrackingType.TRACKED_WITH_ERROR_SESSION_REPLAY + ) + }) + + it('keeps a session on error when the console says so, over what init said', () => { + storeRemoteConfigValues({ sessionSampleRate: 0, sessionOnError: true }) + + startRumSessionManagerWithDefaults({ + configuration: { + sessionSampleRate: 100, + sessionReplaySampleRate: 0, + sessionOnError: false, + remoteConfig: REMOTE_SAMPLING_SETUP, + }, + }) + document.dispatchEvent(createNewEvent(DOM_EVENT.CLICK)) + + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe( + RumTrackingType.TRACKED_ON_ERROR_WITHOUT_SESSION_REPLAY + ) + }) + + it('turns the replay-on-error switch off when the console says so', () => { + storeRemoteConfigValues({ sessionReplayOnError: false }) + + startRumSessionManagerWithDefaults({ + configuration: { + sessionSampleRate: 100, + sessionReplaySampleRate: 0, + sessionReplayOnError: true, + remoteConfig: REMOTE_SAMPLING_SETUP, + }, + }) + document.dispatchEvent(createNewEvent(DOM_EVENT.CLICK)) + + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe(RumTrackingType.TRACKED_WITHOUT_SESSION_REPLAY) + }) + + it('turns the session-on-error switch off when the console says so', () => { + storeRemoteConfigValues({ sessionOnError: false }) + + startRumSessionManagerWithDefaults({ + configuration: { + sessionSampleRate: 0, + sessionOnError: true, + remoteConfig: REMOTE_SAMPLING_SETUP, + }, + }) + document.dispatchEvent(createNewEvent(DOM_EVENT.CLICK)) + + // a delivered false must win over init's true, so nothing is collected - not fall back to it + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe(RumTrackingType.NOT_TRACKED) + }) + it('falls back to the rate passed to init for a knob the console did not set', () => { storeRemoteConfigValues({ sessionReplaySampleRate: 100 }) @@ -315,6 +386,21 @@ describe('rum session manager', () => { expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe(RumTrackingType.TRACKED_WITH_SESSION_REPLAY) }) + it('draws a callback-excluded visitor to nothing, past the on-error switch', () => { + // The callback's contract is "0 never collects". A visitor it excludes must not be kept by the + // on-error switch either, or excluding them would quietly become collecting them on error. + startRumSessionManagerWithDefaults({ + configuration: { + sessionSampleRate: 100, + sessionOnError: true, + beforeSampling: () => ({ sessionSampleRate: 0 }), + }, + }) + document.dispatchEvent(createNewEvent(DOM_EVENT.CLICK)) + + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe(RumTrackingType.NOT_TRACKED) + }) + it('receives the delivered rates and custom values', () => { storeRemote({ sessionSampleRate: 42, custom: { viplist: ['u-1'] } }) const beforeSampling = jasmine.createSpy('beforeSampling') @@ -485,6 +571,29 @@ describe('rum session manager', () => { }) }) + it('reports a zero session sample rate for a session kept only because it errors', () => { + // 99 is above any rate below 100, so the plain draw misses and the switch keeps the session + spyOn(Math, 'random').and.returnValue(0.99) + storeRemote({ version: 7, sessionSampleRate: 50, sessionReplaySampleRate: 0 }) + + const rumSessionManager = startRumSessionManagerWithDefaults({ + configuration: { + sessionSampleRate: 50, + sessionOnError: true, + remoteConfig: REMOTE_SAMPLING_SETUP, + drawStoreKey: DRAW_KEY, + }, + }) + document.dispatchEvent(createNewEvent(DOM_EVENT.CLICK)) + + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe( + RumTrackingType.TRACKED_ON_ERROR_WITHOUT_SESSION_REPLAY + ) + // It was kept by the switch, not by the 50% draw it missed, so it stands for one session, not + // 100/50. Reporting the plain rate would have the adoption panel count it as two. + expect(rumSessionManager.findTrackedSession()!.drawnConfiguration!.sessionSampleRate).toBe(0) + }) + it('reports the rate beforeSampling decided, not the delivered one', () => { storeRemote({ version: 3, sessionSampleRate: 0, sessionReplaySampleRate: 0 }) @@ -979,6 +1088,45 @@ describe('rum session manager', () => { }) }) + describe('a session the on-error switch keeps', () => { + it('does not end an on-error session when the rate is zero, because the switch still collects it', () => { + // The switch's own documented shape: the plain rate misses every session, `sessionOnError` + // keeps the ones that error. A zero rate here is that setting, not a stop - ending the + // session would discard exactly what the switch exists to keep, and blind the page from this + // fetch (which lands on every fresh profile and after every deploy) until the first click. + startWith({ sessionSampleRate: 0, sessionOnError: true }) + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).not.toBe(RumTrackingType.NOT_TRACKED) + + deliver({ version: 1, sessionSampleRate: 0 }) + + expect(isSessionEnded()).toBeFalse() + }) + + it('still ends the session when the console turns the switch off at a zero rate', () => { + // The emergency stop is preserved: a rate of zero with the switch explicitly off collects + // nothing, so the running session is decided against and ended. + startWith({ sessionSampleRate: 0, sessionOnError: true }) + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).not.toBe(RumTrackingType.NOT_TRACKED) + + deliver({ version: 1, sessionSampleRate: 0, sessionOnError: false }) + + expect(isSessionEnded()).toBeTrue() + }) + + it('still ends a plainly drawn session at a zero rate even when the switch is on', () => { + // The switch keeps the sessions the plain draw missed; it does not exempt one already + // collected in full. A rate-0 emergency stop still ends this plainly sampled session, which + // then redraws as an on-error one on the visitor's next action. + storeRemote({ version: 1, sessionSampleRate: 100, sessionReplaySampleRate: 0 }) + startWith({ sessionSampleRate: 100, sessionReplaySampleRate: 0, sessionOnError: true }) + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe(RumTrackingType.TRACKED_WITHOUT_SESSION_REPLAY) + + deliver({ version: 2, sessionSampleRate: 0, sessionOnError: true }) + + expect(isSessionEnded()).toBeTrue() + }) + }) + describe('everything else waits for the next session', () => { it('leaves the session alone when the rate moves to a value it cannot decide on', () => { storeRemote({ version: 1, sessionSampleRate: 100 }) @@ -1363,6 +1511,320 @@ describe('rum session manager', () => { }) }) + describe('session replay on error', () => { + for (const mark of ['error', 'force'] as const) { + for (const replacement of [false, true]) { + it(`reconciles ${mark} after lock exhaustion only for its original session (replacement=${replacement})`, () => { + if (!isChromium()) { + pending('requires a cookie store lock') + } + const manager = startRumSessionManagerWithDefaults({ + configuration: { sessionSampleRate: 100, sessionReplaySampleRate: 0, sessionReplayOnError: true }, + }) + const id = manager.findTrackedSession()!.id + const state = `id=${id}&rum=3&created=${Date.now()}&expire=${Date.now() + DURATION}` + setCookie(SESSION_STORE_KEY, `lock=other-tab&${state}`, DURATION) + if (mark === 'error') { + manager.setSessionHasError(id) + } else { + manager.setForcedReplay() + } + clock.tick(1500) + setCookie(SESSION_STORE_KEY, replacement ? state.replace(id, 'replacement') : state, DURATION) + clock.tick(3000) + expect(getSessionState(SESSION_STORE_KEY)[mark === 'error' ? 'hasError' : 'forcedReplay']).toBe( + replacement ? undefined : '1' + ) + }) + } + } + + for (const force of ['setForcedReplay', 'setForcedSession'] as const) { + it(`${force} releases in memory before a locked store can persist it`, () => { + if (!isChromium()) { + pending('requires a cookie store lock') + } + const manager = startRumSessionManagerWithDefaults({ + configuration: { sessionSampleRate: 100, sessionReplaySampleRate: 0, sessionReplayOnError: true }, + }) + const id = manager.findTrackedSession()!.id + setCookie( + SESSION_STORE_KEY, + `lock=other-tab&id=${id}&rum=3&created=${Date.now()}&expire=${Date.now() + DURATION}`, + DURATION + ) + manager[force]() + expect(manager.findTrackedSession()!.sessionReplay).toBe(SessionReplayState.FORCED) + expect(getSessionState(SESSION_STORE_KEY).forcedReplay).toBeUndefined() + }) + + it(`${force} never writes its deferred mark into a replacement session`, () => { + if (!isChromium()) { + pending('requires a cookie store lock') + } + const manager = startRumSessionManagerWithDefaults({ + configuration: { sessionSampleRate: 100, sessionReplaySampleRate: 0, sessionReplayOnError: true }, + }) + const id = manager.findTrackedSession()!.id + setCookie( + SESSION_STORE_KEY, + `lock=other-tab&id=${id}&rum=3&created=${Date.now()}&expire=${Date.now() + DURATION}`, + DURATION + ) + manager[force]() + setCookie( + SESSION_STORE_KEY, + `id=replacement&rum=3&created=${Date.now()}&expire=${Date.now() + DURATION}`, + DURATION + ) + clock.tick(20) + expect(getSessionState(SESSION_STORE_KEY).forcedReplay).toBeUndefined() + }) + } + + it('applies the error-replay type only when the plain replay draw missed', () => { + startRumSessionManagerWithDefaults({ + configuration: { sessionSampleRate: 100, sessionReplaySampleRate: 100, sessionReplayOnError: true }, + }) + + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe(RumTrackingType.TRACKED_WITH_SESSION_REPLAY) + }) + + it('stores the error-replay type when only the switch applies', () => { + startRumSessionManagerWithDefaults({ + configuration: { sessionSampleRate: 100, sessionReplaySampleRate: 0, sessionReplayOnError: true }, + }) + + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe( + RumTrackingType.TRACKED_WITH_ERROR_SESSION_REPLAY + ) + }) + + it('withholds the replay until the session reports an error', () => { + const sessionManager = startRumSessionManagerWithDefaults({ + configuration: { sessionSampleRate: 100, sessionReplaySampleRate: 0, sessionReplayOnError: true }, + }) + + expect(sessionManager.findTrackedSession()!.sessionReplay).toBe(SessionReplayState.BUFFERED_ON_ERROR) + + sessionManager.setSessionHasError(sessionManager.findTrackedSession()!.id) + + expect(sessionManager.findTrackedSession()!.sessionReplay).toBe(SessionReplayState.SAMPLED) + }) + + it('does not mark a session that has since been replaced by another one', () => { + const sessionManager = startRumSessionManagerWithDefaults({ + configuration: { sessionSampleRate: 100, sessionReplaySampleRate: 0, sessionReplayOnError: true }, + }) + + // another tab renewed the session while the mark was on its way to the store + setCookie(SESSION_STORE_KEY, 'id=other-session&rum=3', DURATION) + + sessionManager.setSessionHasError('a-session-that-is-gone') + + expect(getSessionState(SESSION_STORE_KEY).hasError).toBeUndefined() + }) + + it('releases the replay before the store write lands, since that write can be deferred', () => { + if (!isChromium()) { + pending('the store lock, and so a deferred write, only exists on Chromium') + } + const sessionManager = startRumSessionManagerWithDefaults({ + configuration: { sessionSampleRate: 100, sessionReplaySampleRate: 0, sessionReplayOnError: true }, + }) + const sessionId = sessionManager.findTrackedSession()!.id + + // another tab holds the store lock, so the write is deferred through retries + setCookie(SESSION_STORE_KEY, `lock=other-tab&id=${sessionId}&rum=3`, DURATION) + + sessionManager.setSessionHasError(sessionId) + + expect(getSessionState(SESSION_STORE_KEY).hasError).toBeUndefined() + // and yet the buffer must already see it as released: the page or the session may end before + // the write ever lands, and the buffer would otherwise be thrown away + expect(sessionManager.findTrackedSession()!.sessionReplay).toBe(SessionReplayState.SAMPLED) + }) + + it('keeps the released state across a page load, since it is persisted in the session store', () => { + setCookie( + SESSION_STORE_KEY, + `id=abcdef&rum=3&hasError=1&created=${Date.now()}&expire=${Date.now() + DURATION}`, + DURATION + ) + + const sessionManager = startRumSessionManagerWithDefaults() + + expect(sessionManager.findTrackedSession()!.sessionReplay).toBe(SessionReplayState.SAMPLED) + }) + + it('marks the session so a replay kept only because it errored can be told apart', () => { + const sessionManager = startRumSessionManagerWithDefaults({ + configuration: { sessionSampleRate: 100, sessionReplaySampleRate: 0, sessionReplayOnError: true }, + }) + + expect(sessionManager.findTrackedSession()!.sampledOnErrorReplay).toBeTrue() + + // still true once released, so what was stored can be told apart afterwards + sessionManager.setSessionHasError(sessionManager.findTrackedSession()!.id) + + expect(sessionManager.findTrackedSession()!.sampledOnErrorReplay).toBeTrue() + }) + + it('does not mark a session whose replay is collected unconditionally', () => { + const sessionManager = startRumSessionManagerWithDefaults({ + configuration: { sessionSampleRate: 100, sessionReplaySampleRate: 100 }, + }) + + expect(sessionManager.findTrackedSession()!.sampledOnErrorReplay).toBeFalse() + }) + + it('releases the replay when it is forced, rather than waiting for an error that may never come', () => { + const sessionManager = startRumSessionManagerWithDefaults({ + configuration: { sessionSampleRate: 100, sessionReplaySampleRate: 0, sessionReplayOnError: true }, + }) + expect(sessionManager.findTrackedSession()!.sessionReplay).toBe(SessionReplayState.BUFFERED_ON_ERROR) + + sessionManager.setForcedReplay() + + expect(sessionManager.findTrackedSession()!.sessionReplay).toBe(SessionReplayState.FORCED) + }) + + it('tracks the session even when neither the replay rate nor the switch applies', () => { + const sessionManager = startRumSessionManagerWithDefaults({ + configuration: { sessionSampleRate: 100, sessionReplaySampleRate: 0, sessionReplayOnError: false }, + }) + + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe(RumTrackingType.TRACKED_WITHOUT_SESSION_REPLAY) + expect(sessionManager.findTrackedSession()!.sessionReplay).toBe(SessionReplayState.OFF) + }) + }) + + describe('session on error', () => { + const ON_ERROR_ONLY = { + sessionSampleRate: 0, + sessionOnError: true, + sessionReplaySampleRate: 0, + sessionReplayOnError: false, + } + + it('applies the on-error type only when the plain session draw missed', () => { + startRumSessionManagerWithDefaults({ + configuration: { ...ON_ERROR_ONLY, sessionSampleRate: 100 }, + }) + + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe(RumTrackingType.TRACKED_WITHOUT_SESSION_REPLAY) + }) + + it('withholds the events of a session drawn on error', () => { + const sessionManager = startRumSessionManagerWithDefaults({ configuration: ON_ERROR_ONLY }) + + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe( + RumTrackingType.TRACKED_ON_ERROR_WITHOUT_SESSION_REPLAY + ) + expect(sessionManager.findTrackedSession()!.eventsWithheld).toBeTrue() + + sessionManager.setSessionHasError(sessionManager.findTrackedSession()!.id) + + expect(sessionManager.findTrackedSession()!.eventsWithheld).toBeFalse() + }) + + it('withholds the replay alongside the events, even when the plain replay rate was drawn', () => { + // a replay uploaded while the events are withheld would have no session to attach to + const sessionManager = startRumSessionManagerWithDefaults({ + configuration: { ...ON_ERROR_ONLY, sessionReplaySampleRate: 100 }, + }) + + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe( + RumTrackingType.TRACKED_ON_ERROR_WITH_SESSION_REPLAY + ) + expect(sessionManager.findTrackedSession()!.sessionReplay).toBe(SessionReplayState.BUFFERED_ON_ERROR) + }) + + it('releases events and replay together on the first error', () => { + const sessionManager = startRumSessionManagerWithDefaults({ + configuration: { ...ON_ERROR_ONLY, sessionReplaySampleRate: 100 }, + }) + + sessionManager.setSessionHasError(sessionManager.findTrackedSession()!.id) + + const session = sessionManager.findTrackedSession()! + expect(session.eventsWithheld).toBeFalse() + expect(session.sessionReplay).toBe(SessionReplayState.SAMPLED) + }) + + it('releases a session withholding only its events when the host forces it', () => { + const sessionManager = startRumSessionManagerWithDefaults({ configuration: ON_ERROR_ONLY }) + const sessionId = sessionManager.findTrackedSession()!.id + expect(sessionManager.findTrackedSession()!.eventsWithheld).toBeTrue() + + sessionManager.setForcedSession() + + const session = sessionManager.findTrackedSession()! + // the same session, released, with the replay the host asked for + expect(session.id).toBe(sessionId) + expect(session.eventsWithheld).toBeFalse() + expect(session.sessionReplay).toBe(SessionReplayState.FORCED) + }) + + it('releases the events when capture is forced, so the forced replay is not left orphaned', () => { + const sessionManager = startRumSessionManagerWithDefaults({ + configuration: { ...ON_ERROR_ONLY, sessionReplaySampleRate: 100 }, + }) + + sessionManager.setForcedReplay() + + const session = sessionManager.findTrackedSession()! + expect(session.eventsWithheld).toBeFalse() + expect(session.sessionReplay).toBe(SessionReplayState.FORCED) + }) + + it('draws the type that withholds the replay too when only the on-error replay switch is on', () => { + const sessionManager = startRumSessionManagerWithDefaults({ + configuration: { ...ON_ERROR_ONLY, sessionReplayOnError: true }, + }) + + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe( + RumTrackingType.TRACKED_ON_ERROR_WITH_SESSION_REPLAY + ) + expect(sessionManager.findTrackedSession()!.sessionReplay).toBe(SessionReplayState.BUFFERED_ON_ERROR) + }) + + it('keeps a stored on-error type across a page load rather than drawing again', () => { + setCookie(SESSION_STORE_KEY, `id=abcdef&rum=4&created=${Date.now()}&expire=${Date.now() + DURATION}`, DURATION) + + // a rate that would draw a plainly tracked session, so honouring the stored type is the only + // way this can still be an on-error one + const sessionManager = startRumSessionManagerWithDefaults({ configuration: { sessionSampleRate: 100 } }) + + expect(getSessionState(SESSION_STORE_KEY)[RUM_SESSION_KEY]).toBe( + RumTrackingType.TRACKED_ON_ERROR_WITHOUT_SESSION_REPLAY + ) + expect(sessionManager.findTrackedSession()!.eventsWithheld).toBeTrue() + }) + + it('keeps a released on-error session released across a page load', () => { + setCookie( + SESSION_STORE_KEY, + `id=abcdef&rum=5&hasError=1&created=${Date.now()}&expire=${Date.now() + DURATION}`, + DURATION + ) + + const sessionManager = startRumSessionManagerWithDefaults({ configuration: { sessionSampleRate: 100 } }) + + const session = sessionManager.findTrackedSession()! + expect(session.eventsWithheld).toBeFalse() + expect(session.sampledOnError).toBeTrue() + expect(session.sessionReplay).toBe(SessionReplayState.SAMPLED) + }) + + it('keeps marking the session as on-error once its events have been released', () => { + const sessionManager = startRumSessionManagerWithDefaults({ configuration: ON_ERROR_ONLY }) + sessionManager.setSessionHasError(sessionManager.findTrackedSession()!.id) + + expect(sessionManager.findTrackedSession()!.sampledOnError).toBeTrue() + }) + }) + function startRumSessionManagerWithDefaults({ configuration, trackingConsentState = createTrackingConsentState(TrackingConsent.GRANTED), diff --git a/packages/rum-core/src/domain/rumSessionManager.ts b/packages/rum-core/src/domain/rumSessionManager.ts index 7617bfb0e4..2a994a6008 100644 --- a/packages/rum-core/src/domain/rumSessionManager.ts +++ b/packages/rum-core/src/domain/rumSessionManager.ts @@ -36,6 +36,13 @@ export interface RumSessionManager { expireObservable: Observable setForcedReplay: () => void setForcedSession: () => void + /** + * Marks the given session as having reported an error. This is what releases what an on-error + * session withheld: the replay for a `sessionReplayOnError` session, and the withheld events for a + * `sessionOnError` one. The id is required because the store write can be deferred by the lock, and + * it must not land on a later session. + */ + setSessionHasError: (sessionId: string) => void } /** @@ -80,6 +87,22 @@ export interface DrawnConfiguration { export type RumSession = { id: string sessionReplay: SessionReplayState + /** + * Whether the session collects events but withholds them until it reports an error. Nothing is + * uploaded while this is true, and if the session never errors nothing ever is. + */ + eventsWithheld: boolean + /** + * Whether the session is only kept because of `sessionOnError`. Unlike {@link eventsWithheld} this + * stays true once the error has been reported, so what is stored can be told apart from a plainly + * sampled session - its detail only starts where the buffer reached. + */ + sampledOnError: boolean + /** + * Whether the replay of this session is only kept if it reports an error. Same idea as + * {@link sampledOnError}, for the replay rather than the events. + */ + sampledOnErrorReplay: boolean anonymousId?: string // FLASHCAT FORK - absent when the draw used exactly what init passed — nothing to override then, // the events already report those values — and when the record of the draw did not survive @@ -91,12 +114,21 @@ export const enum RumTrackingType { NOT_TRACKED = '0', TRACKED_WITH_SESSION_REPLAY = '1', TRACKED_WITHOUT_SESSION_REPLAY = '2', + TRACKED_WITH_ERROR_SESSION_REPLAY = '3', + TRACKED_ON_ERROR_WITHOUT_SESSION_REPLAY = '4', + TRACKED_ON_ERROR_WITH_SESSION_REPLAY = '5', } export const enum SessionReplayState { OFF, SAMPLED, FORCED, + /** + * The session records, but every segment is withheld until it reports its first error. If no error + * ever happens, nothing is uploaded and the session is never billed. Once an error is reported the + * session moves to `SAMPLED` and the withheld buffer is released. + */ + BUFFERED_ON_ERROR, } /** @@ -331,8 +363,11 @@ export function startRumSessionManager( } // Asked only now, and only here, because resolving runs the site's `beforeSampling`: this // announcement is not a draw, and the callback should be run no more often than a decision - // actually turns on its answer. - if (resolveSampleRates(configuration, remote).sessionSampleRate > 0) { + // actually turns on its answer. `sessionOnError` counts here too: a session drawn at 0 with + // the switch off is uncollected, and turning the switch on at rate 0 would now keep it on + // error - so it must end and redraw, exactly as a rate rising above 0 makes it. + const nextRates = resolveSampleRates(configuration, remote) + if (nextRates.sessionSampleRate > 0 || nextRates.sessionOnError) { sessionManager.expire() } return @@ -358,7 +393,16 @@ export function startRumSessionManager( return } - if (resolveSampleRates(configuration, remote).sessionSampleRate === 0) { + // FLASHCAT FORK - a rate of zero ends a running session only when nothing else would keep it. + // The exception is this session itself being an on-error one: `sessionOnError` collects exactly + // the sessions the plain rate misses, so a zero rate next to it is the switch's ordinary setting, + // not a stop. Ending such a session would discard the very thing the switch exists to keep, and + // leave the page blind from this fetch until the visitor's first interaction - which is what a + // fresh profile and every deploy would hit on their first configuration fetch. A plainly drawn + // session ('1'/'2'/'3') is still ended by the emergency stop even when the switch is on: the + // switch shapes what the NEXT draw keeps, it does not exempt a session already collected in full. + const { sessionSampleRate, sessionOnError } = resolveSampleRates(configuration, remote) + if (sessionSampleRate === 0 && !(sessionOnError && withholdsEvents(session.trackingType))) { sessionManager.expire() } } @@ -368,12 +412,45 @@ export function startRumSessionManager( endSessionIfSettingsAreDecisive ) - sessionManager.sessionStateUpdateObservable.subscribe(({ previousState, newState }) => { - if (!previousState.forcedReplay && newState.forcedReplay) { - const sessionEntity = sessionManager.findSession() - if (sessionEntity) { - sessionEntity.isReplayForced = true - } + function forceReplay() { + const session = sessionManager.findSession() + if (!session) { + return + } + const wasForced = session.isReplayForced + session.isReplayForced = true + if (!wasForced) { + lifeCycle.notify(LifeCycleEventType.SESSION_RELEASED, { sessionId: session.id, reason: 'force' }) + } + sessionManager.updateSessionState((state) => (state.id === session.id ? { forcedReplay: '1' } : undefined)) + } + + const sessionStateSubscription = sessionManager.sessionStateUpdateObservable.subscribe(({ newState }) => { + const session = sessionManager.findSession() + if (!session || session.id !== newState.id) { + return + } + const becameForced = !session.isReplayForced && newState.forcedReplay === '1' + const becameErrored = !session.hasError && newState.hasError === '1' + session.isReplayForced ||= becameForced + session.hasError ||= becameErrored + if (becameForced || becameErrored) { + lifeCycle.notify(LifeCycleEventType.SESSION_RELEASED, { + sessionId: session.id, + reason: becameForced ? 'force' : 'error', + }) + } + // A lock retry can be exhausted before a mark reaches storage. The existing poll is the + // next opportunity to reconcile it, and the session identity bounds how long it may live. + if ((session.hasError && newState.hasError !== '1') || (session.isReplayForced && newState.forcedReplay !== '1')) { + sessionManager.updateSessionState((state) => + state.id === session.id + ? { + ...(session.hasError ? { hasError: '1' } : {}), + ...(session.isReplayForced ? { forcedReplay: '1' } : {}), + } + : undefined + ) } }) return { @@ -384,12 +461,10 @@ export function startRumSessionManager( } return { id: session.id, - sessionReplay: - session.trackingType === RumTrackingType.TRACKED_WITH_SESSION_REPLAY - ? SessionReplayState.SAMPLED - : session.isReplayForced - ? SessionReplayState.FORCED - : SessionReplayState.OFF, + sessionReplay: computeSessionReplayState(session.trackingType, session.hasError, session.isReplayForced), + eventsWithheld: computeEventsWithheld(session.trackingType, session.hasError, session.isReplayForced), + sampledOnError: withholdsEvents(session.trackingType), + sampledOnErrorReplay: withholdsReplay(session.trackingType), anonymousId: session.anonymousId, // FLASHCAT FORK - looked up at the same time as the session itself, so an event that // belongs to a session already renewed still reports the draw that created it. @@ -399,29 +474,100 @@ export function startRumSessionManager( expire: sessionManager.expire, expireObservable: sessionManager.expireObservable, stop: () => { + sessionStateSubscription.unsubscribe() consentSubscription.unsubscribe() remoteConfigSubscription.unsubscribe() drawnHistory.stop() }, - setForcedReplay: () => sessionManager.updateSessionState({ forcedReplay: '1' }), + setForcedReplay: forceReplay, // FLASHCAT FORK - the escape hatch for "collect this visitor NOW": the host application knows // who needs debugging (its own allow-list, a support flow), the SDK only provides the switch. // A session keeps the decision it was drawn with, so forcing a visitor that was not being // collected means ending their current (empty) session; the next activity draws again with // `forcedSession` set and starts a collected session with replay. A session already collected - // only needs replay forced on, which is the existing forced-replay path. + // only needs replay forced on, which is the existing forced-replay path - and a session that + // withholds its events or its replay until it errors is released the same way, since the host + // asked for it now: forcing the replay is what releases the events too. setForcedSession: () => { forcedSession = true const session = sessionManager.findSession() if (!session || !isTypeTracked(session.trackingType)) { sessionManager.expire() - } else if (session.trackingType === RumTrackingType.TRACKED_WITHOUT_SESSION_REPLAY) { - sessionManager.updateSessionState({ forcedReplay: '1' }) + } else if ( + session.trackingType === RumTrackingType.TRACKED_WITHOUT_SESSION_REPLAY || + withholdsReplay(session.trackingType) || + withholdsEvents(session.trackingType) + ) { + forceReplay() + } + }, + setSessionHasError: (sessionId) => { + const sessionEntity = sessionManager.findSession() + if (sessionEntity?.id === sessionId) { + // Marked in memory straight away, and not only once the store write lands: that write goes + // through a lock that can defer it by several retries, and until then the withheld buffer + // would still read the session as withholding - so an error followed closely by the page or + // the session ending would throw away the very buffer the error was meant to release. + const hadError = sessionEntity.hasError + sessionEntity.hasError = true + if (!hadError) { + lifeCycle.notify(LifeCycleEventType.SESSION_RELEASED, { sessionId, reason: 'error' }) + } } + sessionManager.updateSessionState((state) => (state.id === sessionId ? { hasError: '1' } : undefined)) }, } } +export function withholdsReplay(trackingType: RumTrackingType) { + return ( + trackingType === RumTrackingType.TRACKED_WITH_ERROR_SESSION_REPLAY || + trackingType === RumTrackingType.TRACKED_ON_ERROR_WITH_SESSION_REPLAY + ) +} + +export function withholdsEvents(trackingType: RumTrackingType) { + return ( + trackingType === RumTrackingType.TRACKED_ON_ERROR_WITHOUT_SESSION_REPLAY || + trackingType === RumTrackingType.TRACKED_ON_ERROR_WITH_SESSION_REPLAY + ) +} + +export function computeSessionReplayState( + trackingType: RumTrackingType, + hasError: boolean, + isReplayForced: boolean +): SessionReplayState { + if (trackingType === RumTrackingType.TRACKED_WITH_SESSION_REPLAY) { + return SessionReplayState.SAMPLED + } + if (withholdsReplay(trackingType) && hasError) { + return SessionReplayState.SAMPLED + } + // A forced replay wins over withholding: the host explicitly asked for this user's replay, so it + // must not keep waiting for an error that may never come. + if (isReplayForced) { + return SessionReplayState.FORCED + } + if (withholdsReplay(trackingType)) { + return SessionReplayState.BUFFERED_ON_ERROR + } + return SessionReplayState.OFF +} + +export function computeEventsWithheld( + trackingType: RumTrackingType, + hasError: boolean, + isReplayForced: boolean +): boolean { + // Forcing capture asks for this user's whole session, so it releases the events too - otherwise + // the forced replay would be uploaded for a session that does not exist yet. + if (hasError || isReplayForced) { + return false + } + return withholdsEvents(trackingType) +} + /** * Session id used when the host application does not answer for one, because it was built against * an SDK that predates `getSessionId()`. Such a host is expected to override the session id of the @@ -513,6 +659,10 @@ export function startRumSessionManagerStub( return { id: sessionId ?? STUB_SESSION_ID, sessionReplay, + // The host records for us, or this page uploads what the plain rate drew: neither withholds. + eventsWithheld: false, + sampledOnError: false, + sampledOnErrorReplay: false, anonymousId: bridge?.getAnonymousId(), } }, @@ -520,6 +670,7 @@ export function startRumSessionManagerStub( expireObservable, setForcedReplay: noop, setForcedSession: noop, + setSessionHasError: noop, stop: () => clearInterval(watchIntervalId), } } @@ -552,17 +703,41 @@ function computeSessionState( // the decision it was created with: settings arriving mid-session never start or stop // collecting for a visitor already on the site. const remote = readRemoteConfig(configuration.remoteConfig) - const { sessionSampleRate, sessionReplaySampleRate } = resolveSampleRates(configuration, remote) - - reportDraw(configuration, remote, sessionSampleRate, sessionReplaySampleRate, onDraw) - - if (!performDraw(sessionSampleRate)) { - trackingType = RumTrackingType.NOT_TRACKED - } else if (!performDraw(sessionReplaySampleRate)) { - trackingType = RumTrackingType.TRACKED_WITHOUT_SESSION_REPLAY + const { sessionSampleRate, sessionReplaySampleRate, sessionOnError, sessionReplayOnError } = resolveSampleRates( + configuration, + remote + ) + + if (performDraw(sessionSampleRate)) { + if (performDraw(sessionReplaySampleRate)) { + trackingType = RumTrackingType.TRACKED_WITH_SESSION_REPLAY + } else if (sessionReplayOnError) { + // Only for sessions the plain replay draw missed, so a session is never counted by both. + trackingType = RumTrackingType.TRACKED_WITH_ERROR_SESSION_REPLAY + } else { + trackingType = RumTrackingType.TRACKED_WITHOUT_SESSION_REPLAY + } + } else if (sessionOnError) { + // Only for sessions the plain session draw missed, so a session is never counted by both. + // Such a session never uploads its replay ahead of its events: whichever replay it draws, the + // replay is withheld alongside them, because until they are released the session does not + // exist yet and a replay sent then would have nothing to attach to. + trackingType = + performDraw(sessionReplaySampleRate) || sessionReplayOnError + ? RumTrackingType.TRACKED_ON_ERROR_WITH_SESSION_REPLAY + : RumTrackingType.TRACKED_ON_ERROR_WITHOUT_SESSION_REPLAY } else { - trackingType = RumTrackingType.TRACKED_WITH_SESSION_REPLAY + trackingType = RumTrackingType.NOT_TRACKED } + + // Reported after the ladder, not before, so an on-error session can report the rate the backend + // should extrapolate from. Such a session was kept despite the plain draw missing it, so it + // stands for itself, not for `100 / rate` like a plainly sampled one - reporting the plain rate + // would have the console's adoption panel count each error session as `100 / rate` sessions. A + // rate of 0 there is read as "one session, do not scale". A session merely withholding its + // replay (type '3') was still drawn by the plain rate and reports it unchanged. + const reportedSampleRate = withholdsEvents(trackingType) ? 0 : sessionSampleRate + reportDraw(configuration, remote, reportedSampleRate, sessionReplaySampleRate, onDraw) } return { trackingType, @@ -571,8 +746,10 @@ function computeSessionState( } /** - * FLASHCAT FORK - the rates a draw would use right now: what the console delivered, falling back to - * what the site passed to init, with the application's `beforeSampling` given the last word. This + * FLASHCAT FORK - the rates a draw would use right now, and the on-error switches beside them: what + * the console delivered, falling back to what the site passed to init, with the application's + * `beforeSampling` given the last word on the rates (the switch is not offered to it: it is a + * yes or a no the console already answered). This * is what turns the delivered custom values into sampling decisions without a wasted first draw or * a session restart: the console ships the data (an allow-list, a cohort rule), the application's * own code interprets it here. Its failure modes must never reach session creation, so a thrown @@ -585,6 +762,8 @@ function computeSessionState( function resolveSampleRates(configuration: RumConfiguration, remote: RemoteConfigValues) { let sessionSampleRate = remote.sessionSampleRate ?? configuration.sessionSampleRate let sessionReplaySampleRate = remote.sessionReplaySampleRate ?? configuration.sessionReplaySampleRate + let sessionOnError = remote.sessionOnError ?? configuration.sessionOnError + let sessionReplayOnError = remote.sessionReplayOnError ?? configuration.sessionReplayOnError if (configuration.beforeSampling) { try { @@ -596,9 +775,18 @@ function resolveSampleRates(configuration: RumConfiguration, remote: RemoteConfi if (override) { if (isRate(override.sessionSampleRate)) { sessionSampleRate = override.sessionSampleRate + // The callback's documented contract is "0 never collects". A visitor it draws to 0 must + // not be kept by the on-error switch either, or "never collect" would quietly become + // "collect on error". A rate it leaves alone keeps the switch. + if (override.sessionSampleRate === 0) { + sessionOnError = false + } } if (isRate(override.sessionReplaySampleRate)) { sessionReplaySampleRate = override.sessionReplaySampleRate + if (override.sessionReplaySampleRate === 0) { + sessionReplayOnError = false + } } } } catch (e) { @@ -606,7 +794,12 @@ function resolveSampleRates(configuration: RumConfiguration, remote: RemoteConfi } } - return { sessionSampleRate, sessionReplaySampleRate } + return { + sessionSampleRate, + sessionReplaySampleRate, + sessionOnError, + sessionReplayOnError, + } } /** @@ -756,13 +949,19 @@ function hasValidRumSession(trackingType?: string): trackingType is RumTrackingT return ( trackingType === RumTrackingType.NOT_TRACKED || trackingType === RumTrackingType.TRACKED_WITH_SESSION_REPLAY || - trackingType === RumTrackingType.TRACKED_WITHOUT_SESSION_REPLAY + trackingType === RumTrackingType.TRACKED_WITHOUT_SESSION_REPLAY || + trackingType === RumTrackingType.TRACKED_WITH_ERROR_SESSION_REPLAY || + trackingType === RumTrackingType.TRACKED_ON_ERROR_WITHOUT_SESSION_REPLAY || + trackingType === RumTrackingType.TRACKED_ON_ERROR_WITH_SESSION_REPLAY ) } function isTypeTracked(rumSessionType: RumTrackingType | undefined) { return ( rumSessionType === RumTrackingType.TRACKED_WITHOUT_SESSION_REPLAY || - rumSessionType === RumTrackingType.TRACKED_WITH_SESSION_REPLAY + rumSessionType === RumTrackingType.TRACKED_WITH_SESSION_REPLAY || + rumSessionType === RumTrackingType.TRACKED_WITH_ERROR_SESSION_REPLAY || + rumSessionType === RumTrackingType.TRACKED_ON_ERROR_WITHOUT_SESSION_REPLAY || + rumSessionType === RumTrackingType.TRACKED_ON_ERROR_WITH_SESSION_REPLAY ) } diff --git a/packages/rum-core/src/domain/trackSessionError.spec.ts b/packages/rum-core/src/domain/trackSessionError.spec.ts new file mode 100644 index 0000000000..3b72e12d84 --- /dev/null +++ b/packages/rum-core/src/domain/trackSessionError.spec.ts @@ -0,0 +1,123 @@ +import type { Context } from '@flashcatcloud/browser-core' +import { registerCleanupTask } from '@flashcatcloud/browser-core/test' +import type { RumEvent } from '../rumEvent.types' +import { createRumSessionManagerMock } from '../../test' +import { LifeCycle, LifeCycleEventType } from './lifeCycle' +import { startSessionErrorTracking } from './trackSessionError' + +describe('startSessionErrorTracking', () => { + let lifeCycle: LifeCycle + let sessionManager: ReturnType + let setSessionHasErrorSpy: jasmine.Spy + + function collect(type: string, source = 'source') { + // only error events carry an `error` object; anything else that did would hide a guard that + // reads it before checking the type + const event = type === 'error' ? { type, session: { id: 'session-id' }, error: { source } } : { type } + lifeCycle.notify(LifeCycleEventType.RUM_EVENT_COLLECTED, event as unknown as RumEvent & Context) + } + + beforeEach(() => { + lifeCycle = new LifeCycle() + sessionManager = createRumSessionManagerMock().setTrackedWithErrorSessionReplay() + setSessionHasErrorSpy = spyOn(sessionManager, 'setSessionHasError').and.callThrough() + const { stop } = startSessionErrorTracking(lifeCycle, sessionManager) + registerCleanupTask(stop) + }) + + it('ignores an error from an earlier session without consuming the current session mark', () => { + lifeCycle.notify(LifeCycleEventType.RUM_EVENT_COLLECTED, { + type: 'error', + session: { id: 'previous-session' }, + error: { source: 'custom' }, + } as unknown as RumEvent & Context) + expect(setSessionHasErrorSpy).not.toHaveBeenCalled() + collect('error') + expect(setSessionHasErrorSpy).toHaveBeenCalledOnceWith('session-id') + }) + + it('does not attribute an error without a session id to the current session', () => { + lifeCycle.notify(LifeCycleEventType.RUM_EVENT_COLLECTED, { + type: 'error', + error: { source: 'custom' }, + } as unknown as RumEvent & Context) + expect(setSessionHasErrorSpy).not.toHaveBeenCalled() + }) + + it('marks the session on the first collected error', () => { + collect('error') + + // named, not just counted: the mark is refused if it does not name the session it belongs to + expect(setSessionHasErrorSpy).toHaveBeenCalledOnceWith('session-id') + }) + + it('leaves a session that withholds nothing alone, so an ordinary session store is never written', () => { + sessionManager.setTrackedWithSessionReplay() + + collect('error') + + expect(setSessionHasErrorSpy).not.toHaveBeenCalled() + }) + + it('marks a session that withholds only its events, which has no replay to release', () => { + sessionManager.setTrackedOnError() + + collect('error') + + expect(setSessionHasErrorSpy).toHaveBeenCalledTimes(1) + }) + + it('leaves an untracked session alone', () => { + sessionManager.setNotTracked() + + collect('error') + + expect(setSessionHasErrorSpy).not.toHaveBeenCalled() + }) + + it('does not mark the session on other event types', () => { + collect('view') + collect('resource') + collect('action') + + expect(setSessionHasErrorSpy).not.toHaveBeenCalled() + }) + + it('ignores the SDK own failures, which are not the application reporting an error', () => { + collect('error', 'agent') + + expect(setSessionHasErrorSpy).not.toHaveBeenCalled() + }) + + it('still marks the session on a network error, which is the application reporting one', () => { + collect('error', 'network') + + expect(setSessionHasErrorSpy).toHaveBeenCalledTimes(1) + }) + + it('marks the session only once, however many errors follow', () => { + collect('error') + collect('error') + collect('error') + + expect(setSessionHasErrorSpy).toHaveBeenCalledTimes(1) + }) + + it('marks a renewed session again, since it is a different session', () => { + collect('error') + lifeCycle.notify(LifeCycleEventType.SESSION_RENEWED) + collect('error') + + expect(setSessionHasErrorSpy).toHaveBeenCalledTimes(2) + }) + + it('stops marking once stopped', () => { + const { stop } = startSessionErrorTracking(lifeCycle, sessionManager) + stop() + setSessionHasErrorSpy.calls.reset() + // the suite's own tracker is still running, so exactly one call is expected, not two + collect('error') + + expect(setSessionHasErrorSpy).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/rum-core/src/domain/trackSessionError.ts b/packages/rum-core/src/domain/trackSessionError.ts new file mode 100644 index 0000000000..c431eb9a1d --- /dev/null +++ b/packages/rum-core/src/domain/trackSessionError.ts @@ -0,0 +1,52 @@ +import { ErrorSource } from '@flashcatcloud/browser-core' +import { RumEventType } from '../rawRumEvent.types' +import type { LifeCycle } from './lifeCycle' +import { LifeCycleEventType } from './lifeCycle' +import type { RumSessionManager } from './rumSessionManager' + +/** + * Marks the session as having reported an error, which is what releases what an on-error session + * withheld: a replay withheld by `sessionReplayOnError`, and the events withheld by `sessionOnError`. + * + * It listens after assembly rather than on the raw error, so an error discarded by `beforeSend` or + * by a rate limiter does not release anything: a session billed for an error that cannot be found + * afterwards would be worse than no replay at all. + */ +export function startSessionErrorTracking(lifeCycle: LifeCycle, sessionManager: RumSessionManager) { + let hasReportedError = false + + const eventSubscription = lifeCycle.subscribe(LifeCycleEventType.RUM_EVENT_COLLECTED, (event) => { + if (hasReportedError || event.type !== RumEventType.ERROR) { + return + } + // The SDK's own failures — an intake request that could not be sent, for instance — are ours, + // not the application's. Counting them would turn every session into an error session for any + // customer whose network blocks our endpoint, billing them for replays of nothing. + if (event.error.source === ErrorSource.AGENT) { + return + } + // Only a session that is withholding something has any use for this mark. Setting it on any + // other session would write the session store for customers who enabled neither switch - and that + // write also pushes the session's expiry out (`processSessionStoreOperations` expands every + // state it persists), which would move where their sessions end. + const session = sessionManager.findTrackedSession() + if (!session || event.session?.id !== session.id || (!session.sampledOnError && !session.sampledOnErrorReplay)) { + return + } + hasReportedError = true + sessionManager.setSessionHasError(session.id) + }) + + // A renewed session is a different session: it draws its own sampling and starts out without an + // error, so anything withheld for it must stay withheld until it reports one of its own. + const renewSubscription = lifeCycle.subscribe(LifeCycleEventType.SESSION_RENEWED, () => { + hasReportedError = false + }) + + return { + stop: () => { + eventSubscription.unsubscribe() + renewSubscription.unsubscribe() + }, + } +} diff --git a/packages/rum-core/src/index.ts b/packages/rum-core/src/index.ts index 5aeb6bb7a0..3d23ab451c 100644 --- a/packages/rum-core/src/index.ts +++ b/packages/rum-core/src/index.ts @@ -53,3 +53,4 @@ export type { RumPlugin } from './domain/plugins' export type { MouseEventOnElement } from './domain/action/listenActionEvents' export { supportPerformanceTimingEvent } from './browser/performanceObservable' export { RumPerformanceEntryType } from './browser/performanceObservable' +export { WITHHELD_BUFFER_DURATION } from './transport/withheldEventBuffer' diff --git a/packages/rum-core/src/transport/startRumBatch.spec.ts b/packages/rum-core/src/transport/startRumBatch.spec.ts new file mode 100644 index 0000000000..33c671f783 --- /dev/null +++ b/packages/rum-core/src/transport/startRumBatch.spec.ts @@ -0,0 +1,111 @@ +import { + SESSION_STORE_KEY, + STORAGE_POLL_DELAY, + setCookie, + createTrackingConsentState, + TrackingConsent, + stopSessionManager, + Observable, + createIdentityEncoder, + noop, +} from '@flashcatcloud/browser-core' +import { getSessionState, interceptRequests, mockClock, registerCleanupTask } from '@flashcatcloud/browser-core/test' +import { createRumSessionManagerMock, mockRumConfiguration } from '../../test' +import { LifeCycle, LifeCycleEventType } from '../domain/lifeCycle' +import { startSessionErrorTracking } from '../domain/trackSessionError' +import { startRumSessionManager } from '../domain/rumSessionManager' +import type { RumEvent } from '../rumEvent.types' +import { startRumBatch } from './startRumBatch' + +describe('withheld events through the real batch', () => { + for (const released of [true, false]) { + it(`observes a shared cookie release without a new RUM event (released=${released})`, () => { + const clock = mockClock() + const lifeCycle = new LifeCycle() + const configuration = mockRumConfiguration({ sessionSampleRate: 0, sessionOnError: true }) + const session = startRumSessionManager( + configuration, + lifeCycle, + createTrackingConsentState(TrackingConsent.GRANTED) + ) + const requests = interceptRequests() + const batch = startRumBatch( + configuration, + lifeCycle, + new Observable(), + noop, + new Observable(), + session, + createIdentityEncoder + ) + registerCleanupTask(() => { + batch.stop() + session.stop() + stopSessionManager() + clock.cleanup() + }) + lifeCycle.notify(LifeCycleEventType.RUM_EVENT_COLLECTED, { + type: 'view', + date: 1, + session: { id: session.findTrackedSession()!.id }, + view: { id: 'view-id' }, + } as any) + if (released) { + setCookie( + SESSION_STORE_KEY, + Object.entries({ ...getSessionState(SESSION_STORE_KEY), hasError: '1' }) + .map(([key, value]) => `${key}=${value}`) + .join('&'), + 60000 + ) + } + clock.tick(STORAGE_POLL_DELAY + 3001) + batch.flush('session_expire') + const events = requests.requests.flatMap((request) => + request.body.split('\n').map((line) => JSON.parse(line) as RumEvent) + ) + expect(events.map((event) => event.type)).toEqual(released ? ['view'] : []) + }) + } + + for (const error of [true, false]) { + it(`drains only released events when stopping (error=${error})`, () => { + const clock = mockClock() + const lifeCycle = new LifeCycle() + const session = createRumSessionManagerMock().setTrackedOnError() + const requests = interceptRequests() + const tracker = startSessionErrorTracking(lifeCycle, session) + const batch = startRumBatch( + mockRumConfiguration(), + lifeCycle, + new Observable(), + noop, + new Observable(), + session, + createIdentityEncoder + ) + registerCleanupTask(() => { + tracker.stop() + batch.stop() + clock.cleanup() + }) + const emit = (type: string) => + lifeCycle.notify(LifeCycleEventType.RUM_EVENT_COLLECTED, { + type, + date: 1, + session: { id: 'session-id' }, + view: { id: 'view-id' }, + error: { source: 'custom' }, + } as any) + emit('view') + if (error) { + emit('error') + } + batch.stop() + const events = requests.requests.flatMap((request) => + request.body.split('\n').map((line) => JSON.parse(line) as RumEvent) + ) + expect(events.map((event) => event.type).sort()).toEqual(error ? ['error', 'view'] : []) + }) + } +}) diff --git a/packages/rum-core/src/transport/startRumBatch.ts b/packages/rum-core/src/transport/startRumBatch.ts index 34e62f83d0..c2ab233233 100644 --- a/packages/rum-core/src/transport/startRumBatch.ts +++ b/packages/rum-core/src/transport/startRumBatch.ts @@ -14,9 +14,9 @@ import { } from '@flashcatcloud/browser-core' import type { RumConfiguration } from '../domain/configuration' import type { LifeCycle } from '../domain/lifeCycle' -import { LifeCycleEventType } from '../domain/lifeCycle' +import type { RumSessionManager } from '../domain/rumSessionManager' import { RumEventType } from '../rawRumEvent.types' -import type { RumEvent } from '../rumEvent.types' +import { startWithheldEventBuffer } from './withheldEventBuffer' export function startRumBatch( configuration: RumConfiguration, @@ -24,7 +24,7 @@ export function startRumBatch( telemetryEventObservable: Observable, reportError: (error: RawError) => void, pageMayExitObservable: Observable, - sessionExpireObservable: Observable, + sessionManager: RumSessionManager, createEncoder: (streamId: DeflateEncoderStreamId) => Encoder ) { const replica = configuration.replica @@ -42,10 +42,12 @@ export function startRumBatch( }, reportError, pageMayExitObservable, - sessionExpireObservable + sessionManager.expireObservable ) - lifeCycle.subscribe(LifeCycleEventType.RUM_EVENT_COLLECTED, (serverRumEvent: RumEvent & Context) => { + // Events reach the batch through the buffer, which either forwards them straight away or withholds + // them until the session reports an error. A session that never errors uploads nothing at all. + const withheldEventBuffer = startWithheldEventBuffer(lifeCycle, sessionManager, (serverRumEvent) => { if (serverRumEvent.type === RumEventType.VIEW) { batch.upsert(serverRumEvent, serverRumEvent.view.id) } else { @@ -55,5 +57,13 @@ export function startRumBatch( telemetryEventObservable.subscribe((event) => batch.add(event, isTelemetryReplicationAllowed(configuration))) - return batch + return { + ...batch, + stop: () => { + // Drain released history while the batch is still listening, then flush its final messages. + withheldEventBuffer.stop() + batch.flush('session_expire') + batch.stop() + }, + } } diff --git a/packages/rum-core/src/transport/withheldEventBuffer.spec.ts b/packages/rum-core/src/transport/withheldEventBuffer.spec.ts new file mode 100644 index 0000000000..81a9456819 --- /dev/null +++ b/packages/rum-core/src/transport/withheldEventBuffer.spec.ts @@ -0,0 +1,702 @@ +import type { Context } from '@flashcatcloud/browser-core' +import { ONE_SECOND, PageExitReason } from '@flashcatcloud/browser-core' +import type { Clock } from '@flashcatcloud/browser-core/test' +import { mockClock, registerCleanupTask } from '@flashcatcloud/browser-core/test' +import { createRumSessionManagerMock } from '../../test' +import { RumEventType } from '../rawRumEvent.types' +import type { RumEvent } from '../rumEvent.types' +import { LifeCycle, LifeCycleEventType } from '../domain/lifeCycle' +import { + WITHHELD_BUFFER_BYTES_LIMIT, + WITHHELD_BUFFER_DURATION, + WITHHELD_BUFFER_EVENTS_LIMIT, + WITHHELD_BUFFER_VIEWS_LIMIT, + WITHHELD_BUFFER_RELEASE_MAX_DELAY, + computeReleaseDelay, + startWithheldEventBuffer, +} from './withheldEventBuffer' + +describe('startWithheldEventBuffer', () => { + let clock: Clock + let lifeCycle: LifeCycle + let sessionManager: ReturnType + let forwarded: Array + let stopBuffer: () => void + + function collect(type: RumEventType, overrides: Context = {}) { + const event = { + type, + date: 1234, + view: { id: 'view-1' }, + session: {}, + ...(type === RumEventType.RESOURCE ? { resource: { status_code: 200 } } : {}), + ...overrides, + } as unknown as RumEvent & Context + lifeCycle.notify(LifeCycleEventType.RUM_EVENT_COLLECTED, event) + return event + } + + /** Everything the buffer released, once the release jitter has elapsed. */ + function releasedAfterJitter() { + clock.tick(WITHHELD_BUFFER_RELEASE_MAX_DELAY) + return forwarded + } + + beforeEach(() => { + clock = mockClock() + lifeCycle = new LifeCycle() + forwarded = [] + sessionManager = createRumSessionManagerMock().setTrackedOnError() + const { stop } = startWithheldEventBuffer(lifeCycle, sessionManager, (event) => forwarded.push(event)) + stopBuffer = stop + registerCleanupTask(() => { + stop() + clock.cleanup() + }) + }) + + it('releases immediately when the current session is forced', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE) + sessionManager.setForcedReplay() + lifeCycle.notify(LifeCycleEventType.SESSION_RELEASED, { sessionId: 'session-id', reason: 'force' }) + expect(forwarded.length).toBe(2) + }) + + it('schedules a release learned from another tab without requiring a new event', () => { + collect(RumEventType.VIEW) + sessionManager.setSessionHasError() + lifeCycle.notify(LifeCycleEventType.SESSION_RELEASED, { sessionId: 'session-id', reason: 'error' }) + expect(forwarded.length).toBe(0) + expect(releasedAfterJitter().length).toBe(1) + }) + + it('ignores a release notification for another session', () => { + collect(RumEventType.VIEW) + lifeCycle.notify(LifeCycleEventType.SESSION_RELEASED, { sessionId: 'other-session', reason: 'force' }) + expect(releasedAfterJitter().length).toBe(0) + }) + + it('settles an errored buffer before stopping and does not forward again', () => { + collect(RumEventType.VIEW) + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + stopBuffer() + expect(forwarded.length).toBe(2) + stopBuffer() + expect(releasedAfterJitter().length).toBe(2) + }) + + it('forwards immediately when the session is not withholding', () => { + sessionManager.setTrackedWithSessionReplay() + + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE) + + expect(forwarded.length).toBe(2) + }) + + it('forwards an event collected after the release instead of holding it again', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE) + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + releasedAfterJitter() + const forwardedAfterRelease = forwarded.length + + // The buffer released and cleared; a later event of the same, now-released session must reach + // the batch straight away rather than be held into a fresh hold-then-release cycle. + collect(RumEventType.RESOURCE, { date: 5678 }) + + expect(forwarded.length).toBe(forwardedAfterRelease + 1) + }) + + it('uploads nothing while the session has not reported an error', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE) + collect(RumEventType.ACTION) + clock.tick(30 * ONE_SECOND) + + expect(forwarded.length).toBe(0) + }) + + it('releases the buffer once the session reports an error', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE) + collect(RumEventType.ACTION) + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + + const released = releasedAfterJitter() + expect(released.map((event) => event.type)).toEqual([ + RumEventType.VIEW, + RumEventType.RESOURCE, + RumEventType.ACTION, + RumEventType.ERROR, + ]) + }) + + it('preserves the history and a releasing error larger than the buffer budget', () => { + const view = collect(RumEventType.VIEW) + const resource = collect(RumEventType.RESOURCE) + sessionManager.setSessionHasError() + const error = collect(RumEventType.ERROR, { + error: { source: 'custom', message: 'x'.repeat(WITHHELD_BUFFER_BYTES_LIMIT + 1) }, + }) + + // The oversized error cannot be held, so it goes out first, ahead of the history it precedes; + // the backend orders by client time, so the wire order does not matter. + expect(releasedAfterJitter()).toEqual([error, view, resource]) + }) + + it('still spreads the history behind the jitter when the releasing error is oversized', () => { + const view = collect(RumEventType.VIEW) + const resource = collect(RumEventType.RESOURCE) + sessionManager.setSessionHasError() + const error = collect(RumEventType.ERROR, { + error: { source: 'custom', message: 'x'.repeat(WITHHELD_BUFFER_BYTES_LIMIT + 1) }, + }) + + // Only the oversized error has left so far; releasing the history in this same tick would defeat + // the jitter for exactly the correlated outage it protects against. + expect(forwarded).toEqual([error]) + expect(releasedAfterJitter()).toEqual([error, view, resource]) + }) + + it('does not release a large error while the session is still withholding', () => { + collect(RumEventType.VIEW) + collect(RumEventType.ERROR, { + error: { source: 'agent', message: 'x'.repeat(WITHHELD_BUFFER_BYTES_LIMIT + 1) }, + }) + expect(releasedAfterJitter()).toEqual([]) + }) + + it('keeps only the latest event of a view, since a view event supersedes the ones before it', () => { + collect(RumEventType.VIEW, { documentVersion: 1 }) + collect(RumEventType.VIEW, { documentVersion: 2 }) + collect(RumEventType.VIEW, { documentVersion: 3 }) + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + + const views = releasedAfterJitter().filter((event) => event.type === RumEventType.VIEW) + expect(views.length).toBe(1) + expect((views[0] as unknown as Context).documentVersion).toBe(3) + }) + + it('drops detail that has aged out of the window', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE) + clock.tick(WITHHELD_BUFFER_DURATION + ONE_SECOND) + collect(RumEventType.ACTION) + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + + const types = releasedAfterJitter().map((event) => event.type) + expect(types).not.toContain(RumEventType.RESOURCE) + expect(types).toContain(RumEventType.ACTION) + }) + + it('drops the buffer, and what is still arriving for it, when the session ends without an error', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE) + + sessionManager.setNotTracked() + collect(RumEventType.RESOURCE) + + expect(releasedAfterJitter().length).toBe(0) + }) + + it('forwards the events of a new session that withholds nothing', () => { + sessionManager.setId('session-1') + collect(RumEventType.VIEW, { session: { id: 'session-1' } }) + + sessionManager.setId('session-2').setTrackedWithSessionReplay() + collect(RumEventType.RESOURCE, { session: { id: 'session-2' } }) + + expect(releasedAfterJitter().map((event) => (event.session as Context).id)).toEqual(['session-2']) + }) + + it('releases on page exit when the session errored without the buffer having noticed yet', () => { + // the event arrives synchronously, but the session state behind it is written through a lock + // that can defer the write - so the buffer can still read the session as withholding + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE) + sessionManager.setSessionHasError() + // no further event, so nothing re-reads the session before the page goes + + lifeCycle.notify(LifeCycleEventType.PAGE_MAY_EXIT, { reason: PageExitReason.UNLOADING }) + + expect(forwarded.map((event) => event.type)).toEqual([RumEventType.VIEW, RumEventType.RESOURCE]) + }) + + it('keeps the buffer when the page is only hidden, since it comes back', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE, { date: 111 }) + + lifeCycle.notify(LifeCycleEventType.PAGE_MAY_EXIT, { reason: PageExitReason.HIDDEN }) + expect(forwarded.length).toBe(0) + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + + const dates = releasedAfterJitter().map((event) => event.date) + expect(dates).toContain(111) + }) + + it('drops the buffer when the session ends without ever having errored', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE) + + lifeCycle.notify(LifeCycleEventType.SESSION_EXPIRED) + + expect(releasedAfterJitter().length).toBe(0) + }) + + it('sends a release that is still waiting on jitter when the page is about to go', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE) + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + // still inside the jitter window: the error rides along with the buffer, so nothing left yet + expect(forwarded.length).toBe(0) + + lifeCycle.notify(LifeCycleEventType.PAGE_MAY_EXIT, { reason: PageExitReason.UNLOADING }) + + expect(forwarded.map((event) => event.type)).toEqual([RumEventType.VIEW, RumEventType.RESOURCE, RumEventType.ERROR]) + }) + + it('drops long tasks before actions when it runs out of room', () => { + collect(RumEventType.VIEW) + for (let i = 0; i < WITHHELD_BUFFER_EVENTS_LIMIT; i++) { + collect(RumEventType.LONG_TASK) + } + collect(RumEventType.ACTION) + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + + expect(releasedAfterJitter().map((event) => event.type)).toContain(RumEventType.ACTION) + }) + + it('drops newer long tasks before an older action, by tier rather than by age', () => { + collect(RumEventType.VIEW) + // the action is the oldest detail, so eviction by age would take it first; its tier is above a + // long task's, so tiered eviction must keep it and give up the newer long tasks instead + collect(RumEventType.ACTION, { date: 1 }) + for (let i = 0; i < WITHHELD_BUFFER_EVENTS_LIMIT; i++) { + collect(RumEventType.LONG_TASK, { date: 2 }) + } + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + + // the eviction gives up a long task, not the older action - collapsing the action into the long + // task's tier would take the oldest detail, the action, instead + expect(releasedAfterJitter().map((event) => event.type)).toContain(RumEventType.ACTION) + }) + + it('drops a single event larger than the whole budget instead of evicting the minute for it', () => { + const view = collect(RumEventType.VIEW) + const resource = collect(RumEventType.RESOURCE) + // one action whose context alone exceeds the budget: it can never be part of a released buffer, + // so holding it would evict the history to make room it will never fit into + collect(RumEventType.ACTION, { context: { blob: 'x'.repeat(WITHHELD_BUFFER_BYTES_LIMIT + 1) } }) + + sessionManager.setSessionHasError() + const error = collect(RumEventType.ERROR) + + const released = releasedAfterJitter() + expect(released).toContain(view) + expect(released).toContain(resource) + expect(released).toContain(error) + expect(released.some((event) => event.type === RumEventType.ACTION)).toBeFalse() + }) + + it('never drops errors, however full the buffer gets', () => { + collect(RumEventType.VIEW) + collect(RumEventType.ERROR, { date: 1 }) + for (let i = 0; i < WITHHELD_BUFFER_EVENTS_LIMIT * 2; i++) { + collect(RumEventType.LONG_TASK) + } + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR, { date: 2 }) + + const errors = releasedAfterJitter().filter((event) => event.type === RumEventType.ERROR) + expect(errors.some((event) => event.date === 1)).toBeTrue() + }) + + it('gives up the newest error rather than the first one when only errors are left', () => { + collect(RumEventType.VIEW) + for (let i = 0; i < WITHHELD_BUFFER_EVENTS_LIMIT + 20; i++) { + collect(RumEventType.ERROR, { date: i }) + } + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR, { date: 9999 }) + + const dates = releasedAfterJitter() + .filter((event) => event.type === RumEventType.ERROR) + .map((event) => event.date) + // the first error - the one the session is about - survives + expect(dates).toContain(0) + }) + + it('releases every detail alongside the view it hangs from', () => { + // the backend builds the session row out of view events, so a detail without its view would be + // unreachable however the view came to be missing + for (let i = 0; i < 60; i++) { + collect(RumEventType.VIEW, { view: { id: `view-${i}` } }) + collect(RumEventType.RESOURCE, { view: { id: `view-${i}` } }) + } + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR, { view: { id: 'view-59' } }) + + const released = releasedAfterJitter() + const releasedViewIds = new Set( + released.filter((event) => event.type === RumEventType.VIEW).map((event) => event.view.id) + ) + released + .filter((event) => event.type !== RumEventType.VIEW) + .forEach((event) => expect(releasedViewIds.has(event.view.id)).toBeTrue()) + }) + + it('lets a view go once none of its detail is left inside the window', () => { + collect(RumEventType.VIEW, { view: { id: 'old-view' } }) + collect(RumEventType.RESOURCE, { view: { id: 'old-view' } }) + clock.tick(WITHHELD_BUFFER_DURATION + ONE_SECOND) + collect(RumEventType.VIEW, { view: { id: 'current-view' } }) + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR, { view: { id: 'current-view' } }) + + const releasedViewIds = releasedAfterJitter() + .filter((event) => event.type === RumEventType.VIEW) + .map((event) => event.view.id) + expect(releasedViewIds).not.toContain('old-view') + expect(releasedViewIds).toContain('current-view') + }) + + it('drops a straggler of a session whose buffer was already thrown away', () => { + collect(RumEventType.VIEW, { session: { id: 'session-id' } }) + collect(RumEventType.RESOURCE, { session: { id: 'session-id' } }) + + lifeCycle.notify(LifeCycleEventType.SESSION_EXPIRED) + sessionManager.setNotTracked() + + // a request that started before the session ended completes after it, still carrying its id - + // uploading it would store the very session the withholding was there to avoid + collect(RumEventType.RESOURCE, { session: { id: 'session-id' } }) + + expect(releasedAfterJitter().length).toBe(0) + }) + + it('does not let a straggler of the previous session ride the new one buffer', () => { + sessionManager.setId('session-1') + collect(RumEventType.VIEW, { session: { id: 'session-1' } }) + collect(RumEventType.RESOURCE, { session: { id: 'session-1' } }) + + lifeCycle.notify(LifeCycleEventType.SESSION_EXPIRED) + sessionManager.setId('session-2') + + collect(RumEventType.RESOURCE, { session: { id: 'session-1' } }) + collect(RumEventType.VIEW, { session: { id: 'session-2' } }) + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR, { session: { id: 'session-2' } }) + + const releasedSessionIds = releasedAfterJitter().map((event) => (event.session as Context).id) + expect(releasedSessionIds).toEqual(['session-2', 'session-2']) + }) + + it('keeps the minute before the error when the release timer is held back', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE, { date: 111 }) + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + + // a backgrounded tab clamps timers to about once a minute, so the release runs long after it + // was scheduled - the window it releases has to be the one around the error, not around now + clock.setDate(new Date(Date.now() + WITHHELD_BUFFER_DURATION + ONE_SECOND)) + clock.tick(WITHHELD_BUFFER_RELEASE_MAX_DELAY) + + expect(forwarded.map((event) => event.date)).toContain(111) + }) + + it('still drops a straggler of a session discarded several renewals ago', () => { + sessionManager.setId('session-1') + collect(RumEventType.VIEW, { session: { id: 'session-1' } }) + lifeCycle.notify(LifeCycleEventType.SESSION_EXPIRED) + + sessionManager.setId('session-2') + collect(RumEventType.VIEW, { session: { id: 'session-2' } }) + lifeCycle.notify(LifeCycleEventType.SESSION_EXPIRED) + + sessionManager.setId('session-3') + collect(RumEventType.VIEW, { session: { id: 'session-3' } }) + + // a request that outlived two withheld sessions finally completes + collect(RumEventType.RESOURCE, { session: { id: 'session-1' } }) + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR, { session: { id: 'session-3' } }) + + const releasedSessionIds = releasedAfterJitter().map((event) => (event.session as Context).id) + expect(releasedSessionIds).toEqual(['session-3', 'session-3']) + }) + + it('drops the withheld buffer but keeps uploading a session an older bundle rewrote under the same id', () => { + const view = collect(RumEventType.VIEW, { session: { id: 'session-id' }, date: 1 }) + const resource = collect(RumEventType.RESOURCE, { session: { id: 'session-id' }, date: 2 }) + + // an older SDK sharing the same session store does not know this tracking type and redraws it, + // keeping the id: the session stops withholding, but it never reported an error + sessionManager.setTrackedWithoutSessionReplay() + const resourceAfter = collect(RumEventType.RESOURCE, { session: { id: 'session-id' }, date: 3 }) + const laterResource = collect(RumEventType.RESOURCE, { session: { id: 'session-id' }, date: 4 }) + + const released = releasedAfterJitter() + // what was withheld never earned release and is dropped... + expect(released).not.toContain(view) + expect(released).not.toContain(resource) + // ...but the session is not gone, so its id is not blacklisted and its events go on uploading as + // the plain session it now is - both the one that triggered the discard and the ones after it, + // rather than being dropped for the rest of the session + expect(released).toContain(resourceAfter) + expect(released).toContain(laterResource) + }) + + it('does not blacklist a session an older bundle rewrote under the same id when the expiry arrives first', () => { + collect(RumEventType.VIEW, { session: { id: 'session-id' }, date: 1 }) + collect(RumEventType.RESOURCE, { session: { id: 'session-id' }, date: 2 }) + + // The store poll notices the foreign rewrite before any event of the plain session arrives: + // the session expires with nothing tracked anymore, which discards the buffer and blacklists + // its id... + sessionManager.setNotTracked() + lifeCycle.notify(LifeCycleEventType.SESSION_EXPIRED) + + // ...and the store then renews it under the SAME id, as the plain session the older bundle + // redrew it into + sessionManager.setTrackedWithoutSessionReplay() + const resourceAfter = collect(RumEventType.RESOURCE, { session: { id: 'session-id' }, date: 3 }) + + // The id is blacklisted, but the session wearing it is live and the backend goes on storing + // it: its events must not be dropped for the rest of the session. + expect(forwarded).toEqual([resourceAfter]) + }) + + it('releases the views oldest first, since a session is built out of the first one to arrive', () => { + collect(RumEventType.VIEW, { date: 1000, view: { id: 'view-1' } }) + collect(RumEventType.RESOURCE, { view: { id: 'view-1' } }) + collect(RumEventType.VIEW, { date: 2000, view: { id: 'view-2' } }) + collect(RumEventType.RESOURCE, { view: { id: 'view-2' } }) + collect(RumEventType.VIEW, { date: 3000, view: { id: 'view-3' } }) + collect(RumEventType.RESOURCE, { view: { id: 'view-3' } }) + // a late update of the first view, which puts the oldest view last in the buffer + collect(RumEventType.VIEW, { date: 1000, view: { id: 'view-1' } }) + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR, { view: { id: 'view-3' } }) + + const releasedViewDates = releasedAfterJitter() + .filter((event) => event.type === RumEventType.VIEW) + .map((event) => event.date) + expect(releasedViewDates).toEqual([1000, 2000, 3000]) + }) + + it('spreads the release over the window it computed for this session', () => { + const delay = computeReleaseDelay('session-id') + // the fixture itself has to have something to spread, or this proves nothing + expect(delay).toBeGreaterThan(0) + + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE) + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + + clock.tick(delay - 1) + expect(forwarded.length).toBe(0) + + clock.tick(1) + expect(forwarded.length).toBeGreaterThan(0) + }) + + it('gives up detail once the bytes budget is spent, not only once the count is', () => { + const bulk = 'x'.repeat(8000) + collect(RumEventType.VIEW) + const heldCount = Math.ceil(WITHHELD_BUFFER_BYTES_LIMIT / 8000) + 2 + for (let i = 0; i < heldCount; i++) { + collect(RumEventType.LONG_TASK, { date: i + 1, context: { bulk } }) + } + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + + const releasedLongTasks = releasedAfterJitter().filter((event) => event.type === RumEventType.LONG_TASK) + expect(releasedLongTasks.length).toBeLessThan(heldCount) + }) + + it('gives up requests that succeeded before those that failed', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE, { resource: { status_code: 500 }, date: 500 }) + collect(RumEventType.RESOURCE, { resource: { status_code: 0 }, date: 1 }) + for (let i = 0; i < WITHHELD_BUFFER_EVENTS_LIMIT; i++) { + collect(RumEventType.RESOURCE, { date: 200 }) + } + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + + const releasedDates = releasedAfterJitter().map((event) => event.date) + expect(releasedDates).toContain(500) + expect(releasedDates).toContain(1) + expect(releasedDates.filter((date) => date === 200).length).toBeLessThan(WITHHELD_BUFFER_EVENTS_LIMIT) + }) + + it('keeps no more views than its limit, however many the page goes through', () => { + const viewCount = WITHHELD_BUFFER_VIEWS_LIMIT + 10 + for (let i = 0; i < viewCount; i++) { + collect(RumEventType.VIEW, { date: i + 1, view: { id: `view-${i}` } }) + collect(RumEventType.RESOURCE, { view: { id: `view-${i}` } }) + } + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR, { view: { id: `view-${viewCount - 1}` } }) + + const releasedViews = releasedAfterJitter().filter((event) => event.type === RumEventType.VIEW) + expect(releasedViews.length).toBe(WITHHELD_BUFFER_VIEWS_LIMIT) + }) + + it('drops what has aged out even when the release comes from the page going', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE, { date: 111 }) + // another tab marked the session; this one collects nothing further before the page goes + clock.tick(WITHHELD_BUFFER_DURATION + ONE_SECOND) + sessionManager.setSessionHasError() + + lifeCycle.notify(LifeCycleEventType.PAGE_MAY_EXIT, { reason: PageExitReason.UNLOADING }) + + expect(forwarded.map((event) => event.date)).not.toContain(111) + }) + + it('forwards a straggler of a session that was never withholding', () => { + sessionManager.setId('session-2') + collect(RumEventType.VIEW, { session: { id: 'session-2' } }) + + // a request of an earlier, plainly sampled session completes now: it was never withheld from + // anyone, and dropping it would lose an event of a session that is already stored + collect(RumEventType.RESOURCE, { session: { id: 'session-1' } }) + + expect(forwarded.map((event) => (event.session as Context).id)).toEqual(['session-1']) + }) + + it('sends a release that is still waiting on jitter when the session ends', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE) + sessionManager.setSessionHasError() + collect(RumEventType.ERROR) + expect(forwarded.length).toBe(0) + + lifeCycle.notify(LifeCycleEventType.SESSION_EXPIRED) + + expect(forwarded.map((event) => event.type)).toEqual([RumEventType.VIEW, RumEventType.RESOURCE, RumEventType.ERROR]) + }) + + it('discards an unreleased buffer when stopping', () => { + collect(RumEventType.VIEW) + collect(RumEventType.RESOURCE) + + stopBuffer() + clock.tick(WITHHELD_BUFFER_RELEASE_MAX_DELAY) + + expect(forwarded.length).toBe(0) + }) + + it('keeps the view an error hangs from even when the view cap has to evict one', () => { + const last = WITHHELD_BUFFER_VIEWS_LIMIT - 1 + // a page that has been through exactly as many views as the buffer will hold + for (let i = 0; i <= last; i++) { + collect(RumEventType.VIEW, { date: i + 1, view: { id: `view-${i}` } }) + collect(RumEventType.RESOURCE, { view: { id: `view-${i}` } }) + } + // every earlier view is updated late, which moves each of them behind the current one - so the + // view in progress ends up the oldest entry, and the cap takes from the oldest + for (let i = 0; i < last; i++) { + collect(RumEventType.VIEW, { date: i + 1, view: { id: `view-${i}` } }) + } + // one more late update, for a view old enough to have been dropped already, tips it over the cap + collect(RumEventType.VIEW, { date: 1, view: { id: 'long-gone-view' } }) + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR, { view: { id: `view-${last}` } }) + + expect(releasedAfterJitter().map((event) => event.type)).toContain(RumEventType.ERROR) + }) + + it('keeps the view an error hangs from when a view that already ended is updated late', () => { + collect(RumEventType.VIEW, { date: 1000, view: { id: 'first-view' } }) + collect(RumEventType.VIEW, { date: 2000, view: { id: 'second-view' } }) + // nothing happens in the second view for longer than the window + clock.tick(WITHHELD_BUFFER_DURATION + ONE_SECOND) + // a late update of the view that already ended: it carries that view's start date, so it must + // not become current again - otherwise the view the error hangs from is the one pruned away + collect(RumEventType.VIEW, { date: 1000, view: { id: 'first-view' } }) + + sessionManager.setSessionHasError() + collect(RumEventType.ERROR, { view: { id: 'second-view' } }) + + expect(releasedAfterJitter().map((event) => event.type)).toContain(RumEventType.ERROR) + }) +}) + +describe('computeReleaseDelay', () => { + function randomSessionId() { + const hex = '0123456789abcdef' + let id = '' + for (let i = 0; i < 36; i++) { + id += i === 8 || i === 13 || i === 18 || i === 23 ? '-' : hex[Math.floor(Math.random() * 16)] + } + return id + } + + it('is stable for a given session', () => { + const id = randomSessionId() + + expect(computeReleaseDelay(id)).toBe(computeReleaseDelay(id)) + }) + + it('stays within the release window', () => { + for (let i = 0; i < 1000; i++) { + const delay = computeReleaseDelay(randomSessionId()) + expect(delay).toBeGreaterThanOrEqual(0) + expect(delay).toBeLessThan(WITHHELD_BUFFER_RELEASE_MAX_DELAY) + } + }) + + it('spreads sessions across the window rather than bunching them up', () => { + // session ids are same-length strings over one small alphabet, so a running sum of their + // character codes lands nearly all of them within a few hundred ms of each other - which delays + // the herd instead of spreading it + const bucketCount = 10 + const buckets = new Array(bucketCount).fill(0) + const samples = 10000 + for (let i = 0; i < samples; i++) { + const bucket = Math.floor( + (computeReleaseDelay(randomSessionId()) / WITHHELD_BUFFER_RELEASE_MAX_DELAY) * bucketCount + ) + buckets[bucket] += 1 + } + + buckets.forEach((count) => { + // a flat spread puts 10% in each; allow a wide margin and still catch bunching + expect(count / samples).toBeGreaterThan(0.05) + expect(count / samples).toBeLessThan(0.2) + }) + }) +}) diff --git a/packages/rum-core/src/transport/withheldEventBuffer.ts b/packages/rum-core/src/transport/withheldEventBuffer.ts new file mode 100644 index 0000000000..4adae03ba3 --- /dev/null +++ b/packages/rum-core/src/transport/withheldEventBuffer.ts @@ -0,0 +1,435 @@ +import type { Context, RelativeTime, TimeoutId } from '@flashcatcloud/browser-core' +import { + ONE_KIBI_BYTE, + ONE_SECOND, + addTelemetryDebug, + clearTimeout, + computeBytesCount, + jsonStringify, + relativeNow, + setTimeout, +} from '@flashcatcloud/browser-core' +import type { LifeCycle } from '../domain/lifeCycle' +import { LifeCycleEventType } from '../domain/lifeCycle' +import type { RumSessionManager } from '../domain/rumSessionManager' +import { RumEventType } from '../rawRumEvent.types' +import type { RumEvent } from '../rumEvent.types' + +/** + * How much history a withheld buffer may span, on the event side and on the replay side alike: it is + * one promise to the customer, that an error session shows the minute leading up to the error. The + * replay side also drops and restarts its buffer on it, which bounds what a session that never + * errors holds on to. + */ +export const WITHHELD_BUFFER_DURATION = 60 * ONE_SECOND + +/** Memory bound. Above it the least valuable events are dropped first, see {@link EvictionTier}. */ +export const WITHHELD_BUFFER_BYTES_LIMIT = 64 * ONE_KIBI_BYTE +export const WITHHELD_BUFFER_EVENTS_LIMIT = 200 + +/** + * A view is the container its events hang from: the backend builds the session row out of view + * events, so a detail released without its view would be unreachable. Views are kept out of the + * eviction budget for that reason, and this only bounds pathological single-page navigation counts. + */ +export const WITHHELD_BUFFER_VIEWS_LIMIT = 50 + +/** + * Correlated errors make every client release at the same instant, right when whatever caused them + * is already under strain. Releases are spread over this window instead. + */ +export const WITHHELD_BUFFER_RELEASE_MAX_DELAY = 3 * ONE_SECOND + +/** + * How many thrown-away sessions to remember, so their stragglers are thrown away too. The session + * context history holds a session for up to its maximum length, so a request that outlives this many + * discarded sessions - hours of them - is forwarded after all. What escapes is a lone detail event + * with no view of its own, which has nothing to attach to at the other end; paying for a longer + * memory to catch it would cost more than it saves. + */ +const DISCARDED_SESSIONS_REMEMBERED = 4 + +/** What gets dropped first when the buffer is over budget. Lower goes first. */ +const enum EvictionTier { + /** Long tasks, and requests that succeeded without complaint. */ + FIRST, + /** Actions and vitals: they explain what the user was doing. */ + LAST, + /** + * Errors are the reason the session is kept at all, so they go only once nothing else is left - + * and even then the newest goes first, because the earliest error is the one that releases the + * buffer and the one the session is about. + */ + LAST_RESORT, +} + +interface WithheldEvent { + event: RumEvent & Context + viewId: string + time: RelativeTime + bytes: number + tier: EvictionTier +} + +export function startWithheldEventBuffer( + lifeCycle: LifeCycle, + sessionManager: RumSessionManager, + forward: (event: RumEvent & Context) => void +) { + /** Latest event per view, in insertion order. */ + let views = new Map() + let details: WithheldEvent[] = [] + let bytes = 0 + let currentViewId: string | undefined + let currentViewDate = -Infinity + let withheldForSessionId: string | undefined + /** The sessions whose buffers were thrown away, so their stragglers are thrown away too. */ + const discardedSessionIds: string[] = [] + let releaseTimeoutId: TimeoutId | undefined + /** When the release was scheduled, which is what freezes the window - see {@link prune}. */ + let releaseScheduledAt: RelativeTime | undefined + let droppedCount = 0 + + const eventSubscription = lifeCycle.subscribe(LifeCycleEventType.RUM_EVENT_COLLECTED, (event) => { + const session = sessionManager.findTrackedSession() + // Which session an event belongs to is what the event says, not whichever session is current: + // assembly resolves the session at the event's own start time, so a request or a view update + // that finishes after its session ended still carries that session's id. An event that does not + // say is treated as the current one's, which is how it was handled before there was a buffer. + const eventSessionId = event.session?.id + const isFrom = (sessionId: string | undefined) => eventSessionId === undefined || eventSessionId === sessionId + + if ( + eventSessionId !== undefined && + discardedSessionIds.indexOf(eventSessionId) !== -1 && + session?.id !== eventSessionId + ) { + // Its session ended without ever reporting an error and everything held for it was thrown + // away. Letting a straggler through would store the very session the withholding avoided. + // The blacklist is only enforced against a session that is not the current one: a blacklisted + // id that is nonetheless live can only come from an older bundle that redrew the session + // under the same id - the store poll then expires and blacklists it before any event of the + // plain session arrives, so the branch below never gets to speak for it. That session never + // died and the backend goes on storing it, so its events keep uploading. A session that + // truly ended comes back with a new id, so a real straggler still matches here. + return + } + + if (withheldForSessionId !== undefined && !(session?.id === withheldForSessionId && session.sampledOnError)) { + // The session that was withholding is gone without ever reporting an error, so what it + // collected never earned its way out. Gone covers more than expiry and renewal: the session + // store is shared with every other SDK bundle on the domain, and one that predates these + // tracking types does not recognise them, so it redraws the session and rewrites the type. + // A session that did report an error keeps both its id and its type, and is left alone here. + const wasWithheldFor = withheldForSessionId + // Blacklist the id only when the session truly changed: a straggler of a renewed or expired + // session must be dropped. But an older bundle that rewrote the type kept the SAME id - the + // session is not gone, it just no longer withholds. Blacklisting its id there would drop every + // event of a session the backend goes on storing; instead drop only the buffer and let this + // event and the ones after it upload as the plain session it now is. + const sessionStillPresent = session?.id === wasWithheldFor + discardBuffer(!sessionStillPresent) + if (isFrom(wasWithheldFor) && !sessionStillPresent) { + return + } + } + + if (session?.eventsWithheld && isFrom(session.id)) { + withheldForSessionId = session.id + hold(event) + return + } + + if (withheldForSessionId !== undefined && isFrom(withheldForSessionId)) { + if ( + event.type === RumEventType.ERROR && + computeBytesCount(jsonStringify(event) ?? '') > WITHHELD_BUFFER_BYTES_LIMIT + ) { + // The session has already earned its release. A single error larger than the history + // budget must reach the normal batch, without evicting itself or the history preceding it - + // so it is forwarded straight away rather than held. The history it precedes still leaves + // behind the jitter: releasing it here in the same tick would defeat the anti-thundering-herd + // spread for exactly the correlated outage the jitter exists for. `scheduleRelease` is a + // no-op if the release the mark already scheduled is still pending. + forward(event) + scheduleRelease() + return + } + // Whatever is still withheld here belongs to a session that has just reported its error: the + // guard above ended every other case. This event, typically the error itself, joins what is + // held so that the whole history leaves in order, and behind the same jitter. + hold(event) + scheduleRelease() + return + } + + forward(event) + }) + + /** + * Called when what is held may not get another chance to leave: the page is going away, or the + * session ended (which is also how a withdrawn tracking consent arrives here). + * + * `discardIfUnreleased` says whether the buffer has anything left to wait for. A session that + * ended is over, so what it never released goes no further. A page being hidden is not: it comes + * back, and dropping the minute it had collected would leave the error that follows with nothing. + * + * A session that had already reported its error is released here rather than dropped, and that + * holds when the session ended because consent was withdrawn: everything held was collected while + * consent stood, and the batch has always flushed what it was holding when a session ends. The + * difference this feature makes is the size of that last flush, up to a minute rather than up to + * a batch. Deliberate, and settled - do not turn it into a discard without saying so out loud. + */ + function settleBuffer(discardIfUnreleased: boolean) { + if (withheldForSessionId === undefined) { + return + } + // A release already scheduled goes out now rather than being lost to the jitter window. The + // session is also re-read, because it may have reported its error without the buffer noticing: + // the event arrives synchronously but the state behind it is written through a lock that can + // defer the write, and "an error, then the user leaves" is exactly what this feature is for. + const session = sessionManager.findTrackedSession() + const hasSinceErrored = !!session && session.id === withheldForSessionId && !session.eventsWithheld + + if (releaseTimeoutId !== undefined || hasSinceErrored) { + release() + } else if (discardIfUnreleased) { + discardBuffer() + } + } + + // Kept on a page exit: switching tabs raises one and the page comes straight back, while a page + // that is really unloading takes the buffer with it either way - so there is nothing to gain by + // dropping it, and a minute of history to lose. The replay side reasons the same way. + const sessionReleaseSubscription = lifeCycle.subscribe( + LifeCycleEventType.SESSION_RELEASED, + ({ sessionId, reason }) => { + if (withheldForSessionId !== sessionId) { + return + } + if (reason === 'force') { + release() + } else { + // The local triggering error is collected later in the same synchronous notification. + scheduleRelease() + } + } + ) + const pageMayExitSubscription = lifeCycle.subscribe(LifeCycleEventType.PAGE_MAY_EXIT, () => settleBuffer(false)) + const sessionExpireSubscription = lifeCycle.subscribe(LifeCycleEventType.SESSION_EXPIRED, () => settleBuffer(true)) + + function hold(event: RumEvent & Context) { + if (event.type === RumEventType.VIEW) { + // Upsert: a view event is cumulative, so the latest one supersedes the ones before it. This + // mirrors what the batch already does with view events. The delete is deliberate - setting an + // existing key leaves its insertion order untouched, so without it the oldest entry would be + // the first view seen rather than the least recently updated one. + views.delete(event.view.id) + views.set(event.view.id, event) + // A view event carries its view's start date, so a late update of a view that already ended + // does not make it current again. Letting it would have `prune` drop the view the next error + // hangs from, and the release would then filter that error out of its own buffer. + if (event.date >= currentViewDate) { + currentViewDate = event.date + currentViewId = event.view.id + } + while (views.size > WITHHELD_BUFFER_VIEWS_LIMIT) { + const oldestViewId: string = views.keys().next().value! + if (oldestViewId === currentViewId) { + // The view in progress is the container the error will hang from, which is why `prune` + // spares it too. Late updates of ended views can push it to the front of the map, so it is + // moved to the back here rather than dropped - which, as above, takes a delete. + const currentView = views.get(oldestViewId)! + views.delete(oldestViewId) + views.set(oldestViewId, currentView) + } + views.delete(views.keys().next().value!) + } + prune() + return + } + + const eventBytes = computeBytesCount(jsonStringify(event) ?? '') + if (eventBytes > WITHHELD_BUFFER_BYTES_LIMIT) { + // A single non-error event larger than the whole budget can never be part of a released + // buffer, and holding it would evict the entire preceding minute to make room it will never + // fit into. Drop it and keep the history instead. The releasing error takes the other path, + // above, where it is forwarded on its own without touching the buffer. + droppedCount += 1 + return + } + const held: WithheldEvent = { + event, + viewId: event.view.id, + time: relativeNow(), + bytes: eventBytes, + tier: getEvictionTier(event), + } + details.push(held) + bytes += held.bytes + + prune() + while (details.length > WITHHELD_BUFFER_EVENTS_LIMIT || bytes > WITHHELD_BUFFER_BYTES_LIMIT) { + if (!evictOne()) { + break + } + } + } + + /** Drops what has aged out of the window, so the span kept is the one we promise. */ + function prune() { + // Once a release is scheduled the window stops moving. The timer carrying that release is + // clamped to about once a minute in a background tab, and pruning against a later `now` would + // throw away exactly the minute before the error that the release exists to deliver. + const now = releaseScheduledAt ?? relativeNow() + const oldestAllowed = (now - WITHHELD_BUFFER_DURATION) as RelativeTime + let cutoff = 0 + while (cutoff < details.length && details[cutoff].time < oldestAllowed) { + bytes -= details[cutoff].bytes + droppedCount += 1 + cutoff += 1 + } + if (cutoff > 0) { + details = details.slice(cutoff) + } + + // A view is kept as the container of the detail hanging from it, so once none of its detail is + // left inside the window it has nothing left to contain. Without this the map would grow with + // every route change for as long as the page lives, holding more than the detail budget itself. + // The view in progress always stays: it is the container the error will hang from. + const viewsWithDetail = new Set(details.map((held) => held.viewId)) + views.forEach((_, viewId) => { + if (viewId !== currentViewId && !viewsWithDetail.has(viewId)) { + views.delete(viewId) + } + }) + } + + /** Removes one event of the least valuable tier present. Returns false when there is none left. */ + function evictOne() { + for (const tier of [EvictionTier.FIRST, EvictionTier.LAST]) { + const index = details.findIndex((held) => held.tier === tier) + if (index !== -1) { + evictAt(index) + return true + } + } + + // Only errors are left. One still has to go to stay within budget, and it is the newest: an + // error storm would otherwise push out the first error, which is the one that released the + // buffer and the one the session is really about. + for (let index = details.length - 1; index >= 0; index -= 1) { + if (details[index].tier === EvictionTier.LAST_RESORT) { + evictAt(index) + return true + } + } + return false + } + + function evictAt(index: number) { + bytes -= details[index].bytes + droppedCount += 1 + details.splice(index, 1) + } + + function scheduleRelease() { + if (releaseTimeoutId !== undefined) { + return + } + releaseScheduledAt = relativeNow() + releaseTimeoutId = setTimeout(release, computeReleaseDelay(withheldForSessionId!)) + } + + function release() { + prune() + + // A detail whose view is gone has no container to hang from, so it would be unreachable. + const releasable = details.filter((held) => views.has(held.viewId)) + + // Oldest first. A Map holds its entries in the order they were last updated, which for a burst + // released all at once is not the order the views happened - and a session is built out of + // whichever of its views arrives first, so that one has to be the earliest. + const orderedViews: Array = [] + views.forEach((view) => orderedViews.push(view)) + orderedViews.sort((left, right) => left.date - right.date) + + orderedViews.forEach(forward) + releasable.forEach((held) => forward(held.event)) + + addTelemetryDebug('Error session event buffer released', { + 'buffer.views_count': views.size, + 'buffer.events_count': releasable.length, + 'buffer.dropped_count': droppedCount, + 'buffer.bytes': bytes, + }) + + clearBuffer() + } + + /** Throws the buffer away, and remembers whose it was so its stragglers go the same way. */ + function discardBuffer(blacklist = true) { + if (blacklist && withheldForSessionId !== undefined) { + discardedSessionIds.push(withheldForSessionId) + if (discardedSessionIds.length > DISCARDED_SESSIONS_REMEMBERED) { + discardedSessionIds.shift() + } + } + clearBuffer() + } + + function clearBuffer() { + clearTimeout(releaseTimeoutId) + releaseTimeoutId = undefined + releaseScheduledAt = undefined + views = new Map() + details = [] + bytes = 0 + droppedCount = 0 + currentViewId = undefined + currentViewDate = -Infinity + withheldForSessionId = undefined + } + + return { + stop: () => { + settleBuffer(true) + sessionReleaseSubscription.unsubscribe() + eventSubscription.unsubscribe() + pageMayExitSubscription.unsubscribe() + sessionExpireSubscription.unsubscribe() + }, + } +} + +function getEvictionTier(event: RumEvent): EvictionTier { + switch (event.type) { + case RumEventType.ERROR: + return EvictionTier.LAST_RESORT + case RumEventType.LONG_TASK: + return EvictionTier.FIRST + case RumEventType.RESOURCE: { + // A request that failed is part of how the error happened; one that succeeded rarely is. + // -1 stands for an unknown status code, which is treated like an ordinary success + const statusCode = event.resource?.status_code ?? -1 + return statusCode === 0 || statusCode >= 400 ? EvictionTier.LAST : EvictionTier.FIRST + } + default: + return EvictionTier.LAST + } +} + +/** + * Deterministic per session, so a client always spreads to the same offset. + * + * Multiplicative rather than a running sum: session ids are same-length strings drawn from the same + * small alphabet, so summing their character codes lands almost every session within a few hundred + * milliseconds of the same value - which delays the herd instead of spreading it. + */ +export function computeReleaseDelay(sessionId: string) { + let hash = 0 + for (let i = 0; i < sessionId.length; i += 1) { + hash = Math.imul(hash, 31) + sessionId.charCodeAt(i) + } + return Math.abs(hash) % WITHHELD_BUFFER_RELEASE_MAX_DELAY +} diff --git a/packages/rum-core/test/mockRumSessionManager.ts b/packages/rum-core/test/mockRumSessionManager.ts index 6bf0a1294a..402abb2349 100644 --- a/packages/rum-core/test/mockRumSessionManager.ts +++ b/packages/rum-core/test/mockRumSessionManager.ts @@ -1,12 +1,24 @@ import { Observable } from '@flashcatcloud/browser-core' -import { SessionReplayState, type DrawnConfiguration, type RumSessionManager } from '../src/domain/rumSessionManager' +import { + RumTrackingType, + computeEventsWithheld, + computeSessionReplayState, + withholdsEvents, + withholdsReplay, + type DrawnConfiguration, + type RumSessionManager, +} from '../src/domain/rumSessionManager' export interface RumSessionManagerMock extends RumSessionManager { setId(id: string): RumSessionManagerMock setNotTracked(): RumSessionManagerMock setTrackedWithoutSessionReplay(): RumSessionManagerMock setTrackedWithSessionReplay(): RumSessionManagerMock + setTrackedWithErrorSessionReplay(): RumSessionManagerMock + setTrackedOnError(): RumSessionManagerMock + setTrackedOnErrorWithSessionReplay(): RumSessionManagerMock setForcedReplay(): RumSessionManagerMock + setSessionHasError(): RumSessionManagerMock setDrawnConfiguration(drawn: DrawnConfiguration): RumSessionManagerMock } @@ -14,31 +26,40 @@ const DEFAULT_ID = 'session-id' const enum SessionStatus { TRACKED_WITH_SESSION_REPLAY, TRACKED_WITHOUT_SESSION_REPLAY, + TRACKED_WITH_ERROR_SESSION_REPLAY, + TRACKED_ON_ERROR, + TRACKED_ON_ERROR_WITH_SESSION_REPLAY, NOT_TRACKED, EXPIRED, } +const TRACKING_TYPES: { [key in SessionStatus]?: RumTrackingType } = { + [SessionStatus.TRACKED_WITH_SESSION_REPLAY]: RumTrackingType.TRACKED_WITH_SESSION_REPLAY, + [SessionStatus.TRACKED_WITHOUT_SESSION_REPLAY]: RumTrackingType.TRACKED_WITHOUT_SESSION_REPLAY, + [SessionStatus.TRACKED_WITH_ERROR_SESSION_REPLAY]: RumTrackingType.TRACKED_WITH_ERROR_SESSION_REPLAY, + [SessionStatus.TRACKED_ON_ERROR]: RumTrackingType.TRACKED_ON_ERROR_WITHOUT_SESSION_REPLAY, + [SessionStatus.TRACKED_ON_ERROR_WITH_SESSION_REPLAY]: RumTrackingType.TRACKED_ON_ERROR_WITH_SESSION_REPLAY, +} + export function createRumSessionManagerMock(): RumSessionManagerMock { let id = DEFAULT_ID let sessionStatus: SessionStatus = SessionStatus.TRACKED_WITH_SESSION_REPLAY let forcedReplay: boolean = false + let hasError: boolean = false let drawnConfiguration: DrawnConfiguration | undefined return { findTrackedSession() { - if ( - sessionStatus !== SessionStatus.TRACKED_WITH_SESSION_REPLAY && - sessionStatus !== SessionStatus.TRACKED_WITHOUT_SESSION_REPLAY - ) { + const trackingType = TRACKING_TYPES[sessionStatus] + if (!trackingType) { return undefined } return { id, - sessionReplay: - sessionStatus === SessionStatus.TRACKED_WITH_SESSION_REPLAY - ? SessionReplayState.SAMPLED - : forcedReplay - ? SessionReplayState.FORCED - : SessionReplayState.OFF, + // Derived the same way as in production, so the mock cannot drift from the real state machine + sessionReplay: computeSessionReplayState(trackingType, hasError, forcedReplay), + eventsWithheld: computeEventsWithheld(trackingType, hasError, forcedReplay), + sampledOnError: withholdsEvents(trackingType), + sampledOnErrorReplay: withholdsReplay(trackingType), anonymousId: 'device-123', drawnConfiguration, } @@ -64,10 +85,26 @@ export function createRumSessionManagerMock(): RumSessionManagerMock { sessionStatus = SessionStatus.TRACKED_WITH_SESSION_REPLAY return this }, + setTrackedWithErrorSessionReplay() { + sessionStatus = SessionStatus.TRACKED_WITH_ERROR_SESSION_REPLAY + return this + }, + setTrackedOnError() { + sessionStatus = SessionStatus.TRACKED_ON_ERROR + return this + }, + setTrackedOnErrorWithSessionReplay() { + sessionStatus = SessionStatus.TRACKED_ON_ERROR_WITH_SESSION_REPLAY + return this + }, setForcedReplay() { forcedReplay = true return this }, + setSessionHasError() { + hasError = true + return this + }, setDrawnConfiguration(drawn) { drawnConfiguration = drawn return this diff --git a/packages/rum-legacy/src/domain/sessionStore.spec.ts b/packages/rum-legacy/src/domain/sessionStore.spec.ts index fdeddb47a4..d8ba16b40d 100644 --- a/packages/rum-legacy/src/domain/sessionStore.spec.ts +++ b/packages/rum-legacy/src/domain/sessionStore.spec.ts @@ -29,6 +29,31 @@ describe('session store', () => { deleteSessionCookie() }) + for (const [rum, flag, tracked] of [ + ['3', '', true], + ['4', '', false], + ['5', '', false], + ['4', '&hasError=1', true], + ['5', '&hasError=1', true], + ['4', '&forcedReplay=1', true], + ['5', '&forcedReplay=1', true], + ['0', '&hasError=1', false], + ] as const) { + it(`respects the shared tracking decision rum=${rum}${flag}`, () => { + document.cookie = `${SESSION_COOKIE_NAME}=id=shared-session&rum=${rum}${flag}&created=${Date.now()}&expire=${Date.now() + ONE_MINUTE};path=/` + expect(createSessionStore(100).getOrCreateSession().isTracked).toBe(tracked) + expect(toSessionState(readRawCookie()).rum).toBe(rum) + }) + } + + it('does not carry release marks into a renewed legacy session', () => { + document.cookie = `${SESSION_COOKIE_NAME}=id=old-session&rum=5&hasError=1&forcedReplay=1&created=${Date.now() - ONE_MINUTE}&expire=${Date.now() - 1};path=/` + createSessionStore(100).getOrCreateSession() + const stored = toSessionState(readRawCookie()) + expect(stored.hasError).toBeUndefined() + expect(stored.forcedReplay).toBeUndefined() + }) + it('creates a session with a lowercase uuid', () => { const session = createSessionStore(100).getOrCreateSession() @@ -344,6 +369,18 @@ describe('session store', () => { expect(toSessionState(readRawCookie()).anonymousId).toBe('11111111-bbbb-0000-bbbb-000000000000') }) + it("drops the modern bundle's write lock instead of carrying it", () => { + document.cookie = `${SESSION_COOKIE_NAME}=id=00000000-aaaa-0000-aaaa-000000000000&created=${Date.now()}&expire=${Date.now() + 60000}&rum=2&lock=11111111-bbbb-0000-bbbb-000000000000;path=/` + + createSessionStore(100).getOrCreateSession() + + // A carried lock would outlive its owner - this build renews the cookie for a year on every + // access, and the modern bundle has no stale-lock recovery. Dropping it lets our rewrite + // clear the lock; the other fields of the session are untouched. + expect(readRawCookie()).not.toContain('lock=') + expect(toSessionState(readRawCookie()).id).toBe('00000000-aaaa-0000-aaaa-000000000000') + }) + it('ignores unknown fields injected into the session cookie', () => { document.cookie = `${SESSION_COOKIE_NAME}=id=00000000-aaaa-0000-aaaa-000000000000&created=${Date.now()}&expire=${Date.now() + 60000}&rum=2&evil=payload;path=/` diff --git a/packages/rum-legacy/src/domain/sessionStore.ts b/packages/rum-legacy/src/domain/sessionStore.ts index 60a870a3a7..7897c3f6d7 100644 --- a/packages/rum-legacy/src/domain/sessionStore.ts +++ b/packages/rum-legacy/src/domain/sessionStore.ts @@ -30,6 +30,9 @@ const EXPIRED = '1' const NOT_TRACKED = '0' const TRACKED_WITH_SESSION_REPLAY = '1' const TRACKED_WITHOUT_SESSION_REPLAY = '2' +const TRACKED_WITH_ERROR_SESSION_REPLAY = '3' +const TRACKED_ON_ERROR_WITHOUT_SESSION_REPLAY = '4' +const TRACKED_ON_ERROR_WITH_SESSION_REPLAY = '5' /** * How long a session may be reused without touching the cookie again. @@ -133,7 +136,7 @@ export function createSessionStore(sessionSampleRate: number) { } function toSession(state: SessionState): LegacySession { - // Both tracked values count. This build never writes '1' itself, but both builds share one cookie + // Honor collected and released decisions. This build writes only '0'/'2', but shares one cookie // jar per domain, and IE enterprise site lists routinely put some urls of a site in compatibility // mode and others not. Reading a session the modern bundle started as untracked would silence // this one for the rest of that session's lifetime. @@ -144,7 +147,13 @@ function toSession(state: SessionState): LegacySession { } function isTracked(state: SessionState): boolean { - return state.rum === TRACKED_WITHOUT_SESSION_REPLAY || state.rum === TRACKED_WITH_SESSION_REPLAY + return ( + state.rum === TRACKED_WITHOUT_SESSION_REPLAY || + state.rum === TRACKED_WITH_SESSION_REPLAY || + state.rum === TRACKED_WITH_ERROR_SESSION_REPLAY || + ((state.rum === TRACKED_ON_ERROR_WITHOUT_SESSION_REPLAY || state.rum === TRACKED_ON_ERROR_WITH_SESSION_REPLAY) && + (state.hasError === '1' || state.forcedReplay === '1')) + ) } /** @@ -193,7 +202,7 @@ function isExpired(state: SessionState, now: number): boolean { // `isExpired` belongs to the modern bundle's vocabulary, not to ours, but it has to be listed here // all the same: carried forward as an unknown field it would mark every session this build writes // as expired, and the modern bundle would start a new one on every page load. -const KNOWN_FIELDS = ['id', 'created', 'expire', 'rum', 'isExpired'] +const KNOWN_FIELDS = ['id', 'created', 'expire', 'rum', 'isExpired', 'hasError', 'forcedReplay'] function serialize(state: SessionState): string { const entries: string[] = [] @@ -218,11 +227,21 @@ function serialize(state: SessionState): string { } function deserialize(value: string): SessionState | undefined { + /* + * `lock` is the one field that must NOT be carried the way unknown fields are. It is the modern + * bundle's cross-tab write lock, held only across a synchronous write sequence. Ferried forward + * it would outlive its owner: this build rewrites the cookie on every access and renews it for a + * year, and the modern bundle has no stale-lock recovery, so a carried lock can wedge its session + * store - every write retried and dropped, every new page's init failing on an empty cache - for + * as long as we keep the cookie alive. Dropping it here lets our rewrite clear a stale lock, and + * lets the modern corruption check detect (and retry) a write of ours that lands inside its lock + * window instead of silently accepting the rollback. + */ const state: SessionState = {} const entries = value.split('&') for (let i = 0; i < entries.length; i++) { const match = /^([a-zA-Z]+)=([a-z0-9-]+)$/.exec(entries[i]) - if (match) { + if (match && match[1] !== 'lock') { state[match[1]] = match[2] } } diff --git a/packages/rum/README.md b/packages/rum/README.md index 11198510c8..e5b56ba897 100644 --- a/packages/rum/README.md +++ b/packages/rum/README.md @@ -32,3 +32,35 @@ flashcatRum.init({ [1]: https://docs.flashcat.cloud/zh/flashduty/rum/introduction [2]: https://www.npmjs.com/package/@flashcatcloud/browser-rum + +## Enabling error session collection across pages + +`sessionReplayOnError` needs the full `browser-rum` bundle. The slim and legacy +bundles do not contain a recorder. `sessionOnError` also requires a bundle with +conditional event buffering; the legacy bundle can only honor a shared session +that has already been released by a compatible modern page. + +Before enabling either option in initialization or remote configuration: + +1. Deploy compatible SDK bundles to every page sharing the session cookie, + including other applications and subdomains when cross-subdomain tracking is + enabled. Keep both error-collection options disabled during this deployment. +2. Account for already-open pages and cached application assets. Publishing a new + SDK does not replace JavaScript in those pages. Require those pages to reload, + or defer enablement until incompatible pages no longer share the session store. +3. Verify navigation and concurrent tabs using the deployed bundles. A session + must keep its identity and conditional decision until an error or explicit + force releases it. Verify that sessions without either trigger upload no + conditional data. +4. Enable the options only after that compatibility check. Before rolling back to + an incompatible bundle, disable conditional collection and end or drain the + existing conditional sessions across the affected pages. Disabling an option + alone does not rewrite every running session's decision. + +Older modern bundles recognize only session tracking values `0`, `1`, and `2`. +They can redraw conditional values `3`, `4`, or `5`, causing unexpected collection +or data loss. The compatible legacy reader recognizes `3` and released `4`/`5`, +but it cannot recover history it never recorded. A browser cannot guarantee +cross-page persistence if its shared store stays locked or becomes unavailable +until the page closes; the SDK retries missing marks through its existing session +poll while that same session remains active. diff --git a/packages/rum/src/boot/postStartStrategy.ts b/packages/rum/src/boot/postStartStrategy.ts index c9678c6f21..58e1854666 100644 --- a/packages/rum/src/boot/postStartStrategy.ts +++ b/packages/rum/src/boot/postStartStrategy.ts @@ -87,6 +87,13 @@ export function createPostStartStrategy( return } + if (shouldForceReplay(session!, options)) { + // Applied before the guard below, not after starting: a session that withholds its replay is + // already recording, so the guard would return without ever releasing it - and releasing what + // is held is the whole of what forcing means for such a session. + sessionManager.setForcedReplay() + } + if (isRecordingInProgress(status)) { return } @@ -95,10 +102,6 @@ export function createPostStartStrategy( // Intentionally not awaiting doStart() to keep it asynchronous doStart().catch(monitorError) - - if (shouldForceReplay(session!, options)) { - sessionManager.setForcedReplay() - } } function stop() { @@ -128,5 +131,11 @@ function isRecordingInProgress(status: RecorderStatus) { } function shouldForceReplay(session: RumSession, options?: StartRecordingOptions) { - return options && options.force && session.sessionReplay === SessionReplayState.OFF + return ( + options && + options.force && + // A withheld replay is as much in need of forcing as one that was never sampled: the host asked + // for this user's replay, so it must not go on waiting for an error that may never come. + (session.sessionReplay === SessionReplayState.OFF || session.sessionReplay === SessionReplayState.BUFFERED_ON_ERROR) + ) } diff --git a/packages/rum/src/boot/recorderApi.spec.ts b/packages/rum/src/boot/recorderApi.spec.ts index 2bfeba5311..f0db0dda63 100644 --- a/packages/rum/src/boot/recorderApi.spec.ts +++ b/packages/rum/src/boot/recorderApi.spec.ts @@ -10,6 +10,7 @@ import { mockRumConfiguration, mockViewHistory, } from '../../../rum-core/test' +import { validateAndBuildRumConfiguration } from '../../../rum-core/src/domain/configuration' import type { CreateDeflateWorker } from '../domain/deflate' import { MockWorker } from '../../test' import { resetDeflateWorkerState } from '../domain/deflate' @@ -73,6 +74,40 @@ describe('makeRecorderApi', () => { } describe('recorder boot', () => { + it('starts a remotely selected buffered replay with the built recording default', async () => { + const configuration = validateAndBuildRumConfiguration({ + applicationId: 'app', + clientToken: 'token', + remoteConfigurationEnabled: true, + })! + setupRecorderApi({ + sessionManager: createRumSessionManagerMock().setTrackedWithErrorSessionReplay(), + startSessionReplayRecordingManually: configuration.startSessionReplayRecordingManually, + }) + rumInit() + expect(loadRecorderSpy).toHaveBeenCalledTimes(1) + await collectAsyncCalls(startRecordingSpy, 1) + }) + + it('keeps automatic start intent until a later session enables buffered replay', async () => { + const configuration = validateAndBuildRumConfiguration({ + applicationId: 'app', + clientToken: 'token', + remoteConfigurationEnabled: true, + })! + const sessionManager = createRumSessionManagerMock().setNotTracked() + setupRecorderApi({ + sessionManager, + startSessionReplayRecordingManually: configuration.startSessionReplayRecordingManually, + }) + rumInit() + expect(loadRecorderSpy).not.toHaveBeenCalled() + sessionManager.setTrackedWithErrorSessionReplay() + lifeCycle.notify(LifeCycleEventType.SESSION_RENEWED) + expect(loadRecorderSpy).toHaveBeenCalledTimes(1) + await collectAsyncCalls(startRecordingSpy, 1) + }) + describe('with automatic start', () => { it('starts recording when init() is called', async () => { setupRecorderApi() @@ -186,6 +221,27 @@ describe('makeRecorderApi', () => { expect(setForcedReplaySpy).toHaveBeenCalledTimes(1) }) + it('releases a withheld replay when forced, although it is already recording', async () => { + const setForcedReplaySpy = jasmine.createSpy() + + setupRecorderApi({ + sessionManager: { + ...createRumSessionManagerMock().setTrackedWithErrorSessionReplay(), + setForcedReplay: setForcedReplaySpy, + }, + startSessionReplayRecordingManually: false, + }) + + rumInit() + await collectAsyncCalls(startRecordingSpy, 1) + + // the recording is already running - what forcing asks for here is that what it holds stops + // waiting for an error + recorderApi.start({ force: true }) + + expect(setForcedReplaySpy).toHaveBeenCalledTimes(1) + }) + it('uses the previously created worker if available', async () => { setupRecorderApi({ startSessionReplayRecordingManually: true }) rumInit({ worker: mockWorker }) diff --git a/packages/rum/src/boot/startRecording.spec.ts b/packages/rum/src/boot/startRecording.spec.ts index 1f0cd6f471..ef8412c261 100644 --- a/packages/rum/src/boot/startRecording.spec.ts +++ b/packages/rum/src/boot/startRecording.spec.ts @@ -157,6 +157,47 @@ describe('startRecording', () => { expect(requests[0].metadata.records_count).toBe(1 + recordsPerFullSnapshot()) }) + it('sends the withheld replay once its session reports an error', async () => { + sessionManager.setTrackedWithErrorSessionReplay() + setupStartRecording() + + document.body.dispatchEvent(createNewEvent('click', { clientX: 1, clientY: 2 })) + // a page exit while the session is still waiting for an error keeps the buffer rather than + // sending it, so what follows joins the same segment + flushSegment(lifeCycle) + document.body.dispatchEvent(createNewEvent('click', { clientX: 3, clientY: 4 })) + + sessionManager.setSessionHasError() + flushSegment(lifeCycle) + + const requests = await readSentRequests(1) + expect(requestSendSpy).toHaveBeenCalledTimes(1) + // one segment, held since the recording started, carrying everything from before the error + expect(requests[0].metadata.creation_reason).toBe('init') + expect(requests[0].metadata.records_count).toBe(2 + recordsPerFullSnapshot()) + }) + + it('drops a withheld replay when the session stops withholding without having errored', async () => { + sessionManager.setTrackedWithErrorSessionReplay() + setupStartRecording() + + document.body.dispatchEvent(createNewEvent('click', { clientX: 1, clientY: 2 })) + + // an older SDK sharing the same session store does not know this tracking type and redraws it. + // The session stops withholding, but it never reported an error, so what it held is not owed a + // trip to the intake. + sessionManager.setTrackedWithSessionReplay() + changeView(lifeCycle) + + document.body.dispatchEvent(createNewEvent('click', { clientX: 3, clientY: 4 })) + flushSegment(lifeCycle) + + const requests = await readSentRequests(1) + // 'init' would be the withheld segment; the first one to reach the intake is the one created + // after the session stopped withholding + expect(requests[0].metadata.creation_reason).toBe('view_change') + }) + it('restarts sending segments when the session is renewed', async () => { sessionManager.setNotTracked() setupStartRecording() diff --git a/packages/rum/src/boot/startRecording.ts b/packages/rum/src/boot/startRecording.ts index 086852c49d..b00d4e4e94 100644 --- a/packages/rum/src/boot/startRecording.ts +++ b/packages/rum/src/boot/startRecording.ts @@ -1,7 +1,7 @@ import type { RawError, HttpRequest, DeflateEncoder } from '@flashcatcloud/browser-core' -import { createHttpRequest, addTelemetryDebug, canUseEventBridge } from '@flashcatcloud/browser-core' +import { createHttpRequest, addTelemetryDebug, canUseEventBridge, noop } from '@flashcatcloud/browser-core' import type { LifeCycle, ViewHistory, RumConfiguration, RumSessionManager } from '@flashcatcloud/browser-rum-core' -import { LifeCycleEventType } from '@flashcatcloud/browser-rum-core' +import { LifeCycleEventType, SessionReplayState } from '@flashcatcloud/browser-rum-core' import { record } from '../domain/record' import { startSegmentCollection, SEGMENT_BYTES_LIMIT } from '../domain/segmentCollection' @@ -28,6 +28,10 @@ export function startRecording( let addRecord: (record: BrowserRecord) => void + // Assigned once recording has started. Segment collection is created first because `record()` + // emits into it, so the buffer reaches for the snapshot through this holder rather than directly. + let takeSubsequentFullSnapshot: () => void = noop + // FLASHCAT FORK (2/4) - see `sessionReplayDirectUpload` in RumInitConfiguration. // Without the option, records are handed over to the host application through the bridge. With // it, they go through the regular segment collection and are uploaded from this page. @@ -38,7 +42,27 @@ export function startRecording( sessionManager, viewHistory, replayRequest, - encoder + encoder, + { + getWithholdingSessionId: () => { + const session = sessionManager.findTrackedSession() + return session?.sessionReplay === SessionReplayState.BUFFERED_ON_ERROR ? session.id : undefined + }, + isReleased: (sessionId) => { + const session = sessionManager.findTrackedSession() + // Still the same session, still one whose replay is kept only on an error, and no longer + // withholding. The middle condition matters: a session can stop withholding without ever + // erroring - an older SDK sharing the same store does not know these tracking types and + // redraws them - and that is a session ending, not a replay earning its way out. + return ( + !!session && + session.id === sessionId && + session.sampledOnErrorReplay && + session.sessionReplay !== SessionReplayState.BUFFERED_ON_ERROR + ) + }, + restartFromFullSnapshot: () => takeSubsequentFullSnapshot(), + } ) addRecord = segmentCollection.addRecord cleanupTasks.push(segmentCollection.stop) @@ -57,13 +81,14 @@ export function startRecording( sessionManager.findTrackedSession()?.drawnConfiguration?.defaultPrivacyLevel ?? configuration.defaultPrivacyLevel, } - const { stop: stopRecording } = record({ + const recording = record({ emit: addRecord, configuration: recordConfiguration, lifeCycle, viewHistory, }) - cleanupTasks.push(stopRecording) + takeSubsequentFullSnapshot = recording.takeSubsequentFullSnapshot + cleanupTasks.push(recording.stop) return { stop: () => { diff --git a/packages/rum/src/domain/getSessionReplayLink.ts b/packages/rum/src/domain/getSessionReplayLink.ts index 1bb7c38ea5..e8df168276 100644 --- a/packages/rum/src/domain/getSessionReplayLink.ts +++ b/packages/rum/src/domain/getSessionReplayLink.ts @@ -34,6 +34,11 @@ function getErrorType(session: RumSession | undefined, isRecordingStarted: boole // - replay sampled out return 'incorrect-session-plan' } + if (session.sessionReplay === SessionReplayState.BUFFERED_ON_ERROR) { + // the session records, but nothing has been uploaded yet and nothing may ever be: there is no + // replay to link to until the session reports an error + return 'replay-not-started' + } if (!isRecordingStarted) { return 'replay-not-started' } diff --git a/packages/rum/src/domain/record/record.ts b/packages/rum/src/domain/record/record.ts index 82e41cb1d6..c7187f1a44 100644 --- a/packages/rum/src/domain/record/record.ts +++ b/packages/rum/src/domain/record/record.ts @@ -33,6 +33,11 @@ export interface RecordOptions { export interface RecordAPI { stop: () => void flushMutations: () => void + /** + * Re-serializes the document so that the records that follow are replayable on their own. Needed + * when a withheld replay buffer is dropped, since it takes its full snapshot with it. + */ + takeSubsequentFullSnapshot: () => void shadowRootsController: ShadowRootsController } @@ -54,7 +59,7 @@ export function record(options: RecordOptions): RecordAPI { const shadowRootsController = initShadowRootsController(configuration, emitAndComputeStats, elementsScrollPositions) - const { stop: stopFullSnapshots } = startFullSnapshots( + const { stop: stopFullSnapshots, takeSubsequentFullSnapshot } = startFullSnapshots( elementsScrollPositions, shadowRootsController, lifeCycle, @@ -95,6 +100,7 @@ export function record(options: RecordOptions): RecordAPI { stopFullSnapshots() }, flushMutations, + takeSubsequentFullSnapshot, shadowRootsController, } } diff --git a/packages/rum/src/domain/record/startFullSnapshots.ts b/packages/rum/src/domain/record/startFullSnapshots.ts index 0e57d03ade..8fc0a64228 100644 --- a/packages/rum/src/domain/record/startFullSnapshots.ts +++ b/packages/rum/src/domain/record/startFullSnapshots.ts @@ -97,5 +97,19 @@ export function startFullSnapshots( unsubscribeViewCreated() unsubscribeReactivated() }, + /** + * Re-serializes the document so that what follows is replayable on its own. Used when a withheld + * replay buffer is dropped: the records kept afterwards need a full snapshot to start from. + */ + takeSubsequentFullSnapshot: () => { + flushMutations() + fullSnapshotCallback( + takeFullSnapshot(timeStampNow(), { + shadowRootsController, + status: SerializationContextStatus.SUBSEQUENT_FULL_SNAPSHOT, + elementsScrollPositions, + }) + ) + }, } } diff --git a/packages/rum/src/domain/replayStats.ts b/packages/rum/src/domain/replayStats.ts index 76c5273f6c..1dc2370f60 100644 --- a/packages/rum/src/domain/replayStats.ts +++ b/packages/rum/src/domain/replayStats.ts @@ -19,6 +19,33 @@ export function addWroteData(viewId: string, additionalBytesCount: number) { getOrCreateReplayStats(viewId).segments_total_raw_size += additionalBytesCount } +/** + * Gives back the segment count {@link addSegment} took, and with it the `index_in_view` the segment + * was holding. Segment collection serializes encoder operations, so a dropped segment returns its + * reservation after the release decision and before the next segment is created. + */ +export function removeSegment(viewId: string) { + const replayStats = statsPerView?.get(viewId) + if (!replayStats) { + return + } + replayStats.segments_count = Math.max(0, replayStats.segments_count - 1) +} + +/** + * Rolls back what a dropped segment's records contributed. These are the counters reported on view + * events, and a withheld segment that is dropped never reached the intake, so it must leave no + * trace in them. + */ +export function discardSegmentData(viewId: string, rawBytesCount: number, recordsCount: number) { + const replayStats = statsPerView?.get(viewId) + if (!replayStats) { + return + } + replayStats.records_count = Math.max(0, replayStats.records_count - recordsCount) + replayStats.segments_total_raw_size = Math.max(0, replayStats.segments_total_raw_size - rawBytesCount) +} + export function getReplayStats(viewId: string) { return statsPerView?.get(viewId) } diff --git a/packages/rum/src/domain/segmentCollection/segmentCollection.spec.ts b/packages/rum/src/domain/segmentCollection/segmentCollection.spec.ts index fe528a3661..a49666cc58 100644 --- a/packages/rum/src/domain/segmentCollection/segmentCollection.spec.ts +++ b/packages/rum/src/domain/segmentCollection/segmentCollection.spec.ts @@ -1,7 +1,7 @@ import type { ClocksState, HttpRequest, TimeStamp } from '@flashcatcloud/browser-core' -import { DeflateEncoderStreamId, PageExitReason } from '@flashcatcloud/browser-core' +import { DeflateEncoderStreamId, noop, PageExitReason } from '@flashcatcloud/browser-core' import type { ViewHistory, ViewHistoryEntry, RumConfiguration } from '@flashcatcloud/browser-rum-core' -import { LifeCycle, LifeCycleEventType } from '@flashcatcloud/browser-rum-core' +import { LifeCycle, LifeCycleEventType, WITHHELD_BUFFER_DURATION } from '@flashcatcloud/browser-rum-core' import type { Clock } from '@flashcatcloud/browser-core/test' import { mockClock, registerCleanupTask, restorePageVisibility } from '@flashcatcloud/browser-core/test' import { createRumSessionManagerMock } from '../../../../rum-core/test' @@ -9,6 +9,7 @@ import type { BrowserRecord, SegmentContext } from '../../types' import { RecordType } from '../../types' import { MockWorker, readMetadataFromReplayPayload } from '../../../test' import { createDeflateEncoder } from '../deflate' +import * as replayStats from '../replayStats' import { computeSegmentContext, doStartSegmentCollection, @@ -70,7 +71,8 @@ describe('startSegmentCollection', () => { lifeCycle, () => context, httpRequestSpy, - createDeflateEncoder(configuration, worker, DeflateEncoderStreamId.REPLAY) + createDeflateEncoder(configuration, worker, DeflateEncoderStreamId.REPLAY), + { getWithholdingSessionId: () => undefined, isReleased: () => false, restartFromFullSnapshot: noop } )) registerCleanupTask(() => { @@ -329,3 +331,527 @@ describe('computeSegmentContext', () => { } as any } }) + +describe('startSegmentCollection withholding (error session replay)', () => { + let clock: Clock + let lifeCycle: LifeCycle + let worker: MockWorker + let httpRequestSpy: { + sendOnExit: jasmine.Spy + send: jasmine.Spy + } + let addRecord: (record: BrowserRecord) => void + let withholdingSessionId: string | undefined + let releasedSessionId: string | undefined + let restartFromFullSnapshotSpy: jasmine.Spy<() => void> + let stopCollection: () => void + + function reportError() { + releasedSessionId = withholdingSessionId + withholdingSessionId = undefined + } + + beforeEach(() => { + clock = mockClock() + lifeCycle = new LifeCycle() + worker = new MockWorker() + httpRequestSpy = { sendOnExit: jasmine.createSpy(), send: jasmine.createSpy() } + withholdingSessionId = CONTEXT.session.id + releasedSessionId = undefined + restartFromFullSnapshotSpy = jasmine.createSpy() + replayStats.resetReplayStats() + + const { stop, addRecord: add } = doStartSegmentCollection( + lifeCycle, + () => CONTEXT, + httpRequestSpy, + createDeflateEncoder({} as RumConfiguration, worker, DeflateEncoderStreamId.REPLAY), + { + getWithholdingSessionId: () => withholdingSessionId, + isReleased: (sessionId) => releasedSessionId === sessionId, + restartFromFullSnapshot: restartFromFullSnapshotSpy, + } + ) + addRecord = add + stopCollection = stop + + registerCleanupTask(() => { + stop() + clock.cleanup() + replayStats.resetReplayStats() + }) + }) + + it('releases a checkout still being encoded without reusing its segment index', async () => { + addRecord({ ...RECORD, type: RecordType.FullSnapshot, data: {} } as BrowserRecord) + worker.processAllMessages() + clock.tick(WITHHELD_BUFFER_DURATION) + reportError() + lifeCycle.notify(LifeCycleEventType.RUM_EVENT_COLLECTED, { type: 'error' } as any) + addRecord(RECORD) + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + const metadata = await Promise.all( + httpRequestSpy.send.calls.allArgs().map(([payload]) => readMetadataFromReplayPayload(payload)) + ) + expect(metadata.map((segment) => segment.index_in_view)).toEqual([0, 1]) + expect(metadata[0]?.has_full_snapshot).toBeTrue() + }) + + it('remembers a release if recording ends before the worker answers', () => { + addRecord(RECORD) + worker.processAllMessages() + clock.tick(WITHHELD_BUFFER_DURATION) + reportError() + lifeCycle.notify(LifeCycleEventType.RUM_EVENT_COLLECTED, { type: 'error' } as any) + stopCollection() + releasedSessionId = undefined + worker.processAllMessages() + expect(httpRequestSpy.send).toHaveBeenCalledTimes(1) + }) + + it('drains records and a stop queued behind a released flush', async () => { + addRecord(RECORD) + worker.processAllMessages() + clock.tick(WITHHELD_BUFFER_DURATION) + addRecord(RECORD) + reportError() + stopCollection() + releasedSessionId = undefined + worker.processAllMessages() + const metadata = await Promise.all( + httpRequestSpy.send.calls.allArgs().map(([payload]) => readMetadataFromReplayPayload(payload)) + ) + expect(metadata.map((segment) => segment.index_in_view)).toEqual([0, 1]) + expect(metadata.map((segment) => segment.records_count)).toEqual([1, 1]) + }) + + it('preserves encoder ordering when a new recording starts before the old flush completes', async () => { + const sharedWorker = new MockWorker() + const sharedEncoder = createDeflateEncoder({} as RumConfiguration, sharedWorker, DeflateEncoderStreamId.REPLAY) + const sent: Array[0]> = [] + let released = false + const request = { send: (payload: Parameters[0]) => sent.push(payload), sendOnExit: noop } + const first = doStartSegmentCollection(lifeCycle, () => CONTEXT, request, sharedEncoder, { + getWithholdingSessionId: () => (released ? undefined : CONTEXT.session.id), + isReleased: () => released, + restartFromFullSnapshot: noop, + }) + first.addRecord(RECORD) + clock.tick(WITHHELD_BUFFER_DURATION) + first.addRecord(RECORD) + released = true + first.stop() + const second = doStartSegmentCollection( + new LifeCycle(), + () => ({ ...CONTEXT, session: { id: 'next-session' }, view: { id: 'next-view' } }), + request, + sharedEncoder, + { + getWithholdingSessionId: () => undefined, + isReleased: () => false, + restartFromFullSnapshot: noop, + } + ) + second.addRecord(RECORD) + second.stop() + sharedWorker.processAllMessages() + const segments = await Promise.all( + sent.map( + async (payload) => + JSON.parse(await ((payload.data as FormData).get('segment') as Blob).text()) as { + session: { id: string } + records: BrowserRecord[] + index_in_view: number + } + ) + ) + expect(segments.map((segment) => segment.session.id)).toEqual([ + CONTEXT.session.id, + CONTEXT.session.id, + 'next-session', + ]) + expect(segments.map((segment) => segment.index_in_view)).toEqual([0, 1, 0]) + expect(segments.map((segment) => segment.records.length)).toEqual([1, 1, 1]) + }) + + it('never releases an unfinished flush for a different session', () => { + addRecord(RECORD) + clock.tick(WITHHELD_BUFFER_DURATION) + releasedSessionId = 'different-session' + stopCollection() + worker.processAllMessages() + expect(httpRequestSpy.send).not.toHaveBeenCalled() + }) + + it('does not send anything while the session has not reported an error', () => { + addRecord(RECORD) + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + + expect(httpRequestSpy.send).not.toHaveBeenCalled() + expect(httpRequestSpy.sendOnExit).not.toHaveBeenCalled() + }) + + it('keeps buffering across several duration limits instead of cutting the segment', () => { + addRecord(RECORD) + clock.tick(SEGMENT_DURATION_LIMIT * 3) + addRecord(RECORD) + worker.processAllMessages() + + expect(httpRequestSpy.send).not.toHaveBeenCalled() + // still the same buffer: dropping it would have asked for a fresh full snapshot + expect(restartFromFullSnapshotSpy).not.toHaveBeenCalled() + }) + + it('sends the withheld buffer once the session reports an error', async () => { + addRecord(RECORD) + addRecord(RECORD) + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + expect(httpRequestSpy.send).not.toHaveBeenCalled() + + reportError() + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + + expect(httpRequestSpy.send).toHaveBeenCalledTimes(1) + // the records collected before the error are part of what is sent + expect((await readMetadataFromReplayPayload(httpRequestSpy.send.calls.mostRecent().args[0])).records_count).toBe(2) + }) + + it('keeps the withheld buffer across a page reactivation instead of cutting it', async () => { + addRecord(RECORD) + worker.processAllMessages() + // A reactivation flush must not cut the withheld buffer: cutting drops it, taking the records + // that came before the reactivation with it and leaving the released replay unable to start + // from them. + lifeCycle.notify(LifeCycleEventType.PAGE_REACTIVATED) + worker.processAllMessages() + expect(restartFromFullSnapshotSpy).not.toHaveBeenCalled() + addRecord(RECORD) + + reportError() + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + + const metadata = await Promise.all( + httpRequestSpy.send.calls.allArgs().map(([payload]) => readMetadataFromReplayPayload(payload)) + ) + const totalRecords = metadata.reduce((count, segment) => count + segment.records_count, 0) + // both records survive in what is released; a reactivation cut would have dropped the first one + expect(totalRecords).toBe(2) + }) + + it('wakes the deferred restart from a SESSION_RELEASED event with no accompanying rum event', () => { + restartFromFullSnapshotSpy.and.callFake(() => addRecord(VERY_BIG_RECORD)) + // An oversized snapshot drops the buffer and arms the deferred restart poll, still withholding. + addRecord(VERY_BIG_RECORD) + worker.processAllMessages() + expect(restartFromFullSnapshotSpy).toHaveBeenCalledTimes(1) + + // The session errors and is released, but nothing else happens on the page - no rum event, no + // clock tick. Only the SESSION_RELEASED subscription can wake the restart here. + reportError() + lifeCycle.notify(LifeCycleEventType.SESSION_RELEASED, { sessionId: CONTEXT.session.id } as any) + worker.processAllMessages() + + expect(restartFromFullSnapshotSpy).toHaveBeenCalledTimes(2) + }) + + it('drops the buffer and restarts from a full snapshot once it spans the checkout time', () => { + addRecord(RECORD) + clock.tick(WITHHELD_BUFFER_DURATION) + worker.processAllMessages() + + expect(httpRequestSpy.send).not.toHaveBeenCalled() + expect(restartFromFullSnapshotSpy).toHaveBeenCalledTimes(1) + }) + + it('drops the buffer and restarts from a full snapshot when it grows past the bytes limit', () => { + addRecord(VERY_BIG_RECORD) + worker.processAllMessages() + + expect(httpRequestSpy.send).not.toHaveBeenCalled() + expect(restartFromFullSnapshotSpy).toHaveBeenCalledTimes(1) + }) + + it('does not restart in a hot loop when the full snapshot alone exceeds the bytes limit', () => { + // every restart would blow the limit again straight away on such a document + addRecord(VERY_BIG_RECORD) + worker.processAllMessages() + addRecord(VERY_BIG_RECORD) + worker.processAllMessages() + addRecord(VERY_BIG_RECORD) + worker.processAllMessages() + + expect(restartFromFullSnapshotSpy).toHaveBeenCalledTimes(1) + + clock.tick(SEGMENT_DURATION_LIMIT) + addRecord(VERY_BIG_RECORD) + worker.processAllMessages() + + expect(restartFromFullSnapshotSpy).toHaveBeenCalledTimes(2) + }) + + it('restores a full snapshot after consecutive oversized snapshots and an error', async () => { + restartFromFullSnapshotSpy.and.callFake(() => addRecord(VERY_BIG_RECORD)) + addRecord(VERY_BIG_RECORD) + worker.processAllMessages() + expect(httpRequestSpy.send).not.toHaveBeenCalled() + expect(restartFromFullSnapshotSpy).toHaveBeenCalledTimes(1) + + reportError() + addRecord(RECORD) + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + + expect(httpRequestSpy.send).toHaveBeenCalled() + expect( + (await readMetadataFromReplayPayload(httpRequestSpy.send.calls.first().args[0])).has_full_snapshot + ).toBeTrue() + }) + + it('restores a missing snapshot before an errored page exits during the restart delay', async () => { + restartFromFullSnapshotSpy.and.callFake(() => addRecord(VERY_BIG_RECORD)) + addRecord(VERY_BIG_RECORD) + worker.processAllMessages() + reportError() + addRecord(RECORD) + lifeCycle.notify(LifeCycleEventType.PAGE_MAY_EXIT, { reason: PageExitReason.UNLOADING }) + worker.processAllMessages() + + expect(httpRequestSpy.sendOnExit).toHaveBeenCalled() + expect( + (await readMetadataFromReplayPayload(httpRequestSpy.sendOnExit.calls.first().args[0])).has_full_snapshot + ).toBeTrue() + }) + + it('restores the missing snapshot as soon as an error releases the session', () => { + restartFromFullSnapshotSpy.and.callFake(() => addRecord(VERY_BIG_RECORD)) + addRecord(VERY_BIG_RECORD) + worker.processAllMessages() + reportError() + lifeCycle.notify(LifeCycleEventType.RUM_EVENT_COLLECTED, { type: 'error' } as any) + + expect(restartFromFullSnapshotSpy).toHaveBeenCalledTimes(2) + worker.processAllMessages() + expect(httpRequestSpy.send).toHaveBeenCalled() + }) + + it('cancels the delayed replacement when a new view supplies a snapshot', () => { + restartFromFullSnapshotSpy.and.callFake(() => addRecord(VERY_BIG_RECORD)) + addRecord(VERY_BIG_RECORD) + worker.processAllMessages() + lifeCycle.notify(LifeCycleEventType.VIEW_CREATED, {} as any) + addRecord({ ...VERY_BIG_RECORD, data: {} } as BrowserRecord) + worker.processAllMessages() + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + + expect(restartFromFullSnapshotSpy).toHaveBeenCalledTimes(1) + expect(httpRequestSpy.send).not.toHaveBeenCalled() + }) + + it('does not repeatedly serialize an oversized document while waiting for an error', () => { + restartFromFullSnapshotSpy.and.callFake(() => addRecord(VERY_BIG_RECORD)) + addRecord(VERY_BIG_RECORD) + worker.processAllMessages() + for (let i = 0; i < 4; i++) { + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + } + + expect(restartFromFullSnapshotSpy).toHaveBeenCalledTimes(1) + expect(httpRequestSpy.send).not.toHaveBeenCalled() + }) + + it('cancels a delayed snapshot when recording stops', () => { + restartFromFullSnapshotSpy.and.callFake(() => addRecord(VERY_BIG_RECORD)) + addRecord(VERY_BIG_RECORD) + worker.processAllMessages() + stopCollection() + clock.tick(SEGMENT_DURATION_LIMIT * 2) + worker.processAllMessages() + + expect(restartFromFullSnapshotSpy).toHaveBeenCalledTimes(1) + expect(httpRequestSpy.send).not.toHaveBeenCalled() + }) + + it('keeps the buffer when the page is only hidden, so the replay can still start from its snapshot', () => { + // switching tabs is ordinary; dropping here would take the only full snapshot with it + addRecord(RECORD) + addRecord(RECORD) + lifeCycle.notify(LifeCycleEventType.PAGE_MAY_EXIT, { reason: PageExitReason.HIDDEN }) + worker.processAllMessages() + + expect(httpRequestSpy.sendOnExit).not.toHaveBeenCalled() + expect(restartFromFullSnapshotSpy).not.toHaveBeenCalled() + + reportError() + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + + expect(httpRequestSpy.send).toHaveBeenCalledTimes(1) + }) + + it('sends nothing on page exit for a session that never errored', () => { + addRecord(RECORD) + lifeCycle.notify(LifeCycleEventType.PAGE_MAY_EXIT, { reason: PageExitReason.UNLOADING }) + worker.processAllMessages() + + expect(httpRequestSpy.sendOnExit).not.toHaveBeenCalled() + }) + + it('does not let a dropped buffer leave its index_in_view behind for the next one to collide with', async () => { + // the restart emits records, exactly as taking a fresh full snapshot does in production + restartFromFullSnapshotSpy.and.callFake(() => addRecord(RECORD)) + + addRecord(RECORD) + clock.tick(WITHHELD_BUFFER_DURATION) + worker.processAllMessages() + expect(restartFromFullSnapshotSpy).toHaveBeenCalledTimes(1) + + reportError() + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + + // the dropped buffer never reached the intake, so the first segment that does is index 0 + const metadata = await readMetadataFromReplayPayload(httpRequestSpy.send.calls.mostRecent().args[0]) + expect(metadata.index_in_view).toBe(0) + // and it carries a reason the segment schema knows, not the internal one that dropped the buffer + expect(metadata.creation_reason).toBe('segment_duration_limit') + }) + + it('does not restart the buffer when collection was stopped while the flush was in flight', () => { + addRecord(RECORD) + // the checkout flush is posted to the worker, and recording is stopped before it answers + clock.tick(WITHHELD_BUFFER_DURATION) + stopCollection() + worker.processAllMessages() + + expect(restartFromFullSnapshotSpy).not.toHaveBeenCalled() + }) + + it('does not hand the next segment an index the dropped one still holds when a record lands mid-flush', async () => { + restartFromFullSnapshotSpy.and.callFake(() => addRecord(RECORD)) + + addRecord(RECORD) + // The flush is posted to the worker but not answered yet - in production that round trip always + // happens, because flushing writes the trailer before finishing. A record arriving now creates + // the next segment, which reads its index while the dropped one is still counted. + clock.tick(WITHHELD_BUFFER_DURATION) + addRecord(RECORD) + worker.processAllMessages() + + reportError() + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + + expect((await readMetadataFromReplayPayload(httpRequestSpy.send.calls.mostRecent().args[0])).index_in_view).toBe(0) + }) + + it('leaves no trace of a dropped buffer in the replay stats', () => { + addRecord(RECORD) + clock.tick(WITHHELD_BUFFER_DURATION) + worker.processAllMessages() + + const stats = replayStats.getReplayStats(CONTEXT.view.id) + expect(stats?.segments_count ?? 0).toBe(0) + expect(stats?.segments_total_raw_size ?? 0).toBe(0) + }) + + it('sends normally once released, without withholding the following segments', () => { + reportError() + addRecord(RECORD) + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + addRecord(RECORD) + clock.tick(SEGMENT_DURATION_LIMIT) + worker.processAllMessages() + + expect(httpRequestSpy.send).toHaveBeenCalledTimes(2) + }) +}) + +describe('startSegmentCollection withholding, session lifecycle', () => { + let clock: Clock + let lifeCycle: LifeCycle + let worker: MockWorker + let httpRequestSpy: { + sendOnExit: jasmine.Spy + send: jasmine.Spy + } + let addRecord: (record: BrowserRecord) => void + let stopSegmentCollection: () => void + let withholdingSessionId: string | undefined + let releasedSessionId: string | undefined + + beforeEach(() => { + clock = mockClock() + lifeCycle = new LifeCycle() + worker = new MockWorker() + httpRequestSpy = { sendOnExit: jasmine.createSpy(), send: jasmine.createSpy() } + withholdingSessionId = CONTEXT.session.id + releasedSessionId = undefined + + const { stop, addRecord: add } = doStartSegmentCollection( + lifeCycle, + () => CONTEXT, + httpRequestSpy, + createDeflateEncoder({} as RumConfiguration, worker, DeflateEncoderStreamId.REPLAY), + { + getWithholdingSessionId: () => withholdingSessionId, + isReleased: (sessionId) => releasedSessionId === sessionId, + restartFromFullSnapshot: () => undefined, + } + ) + addRecord = add + stopSegmentCollection = stop + + registerCleanupTask(() => { + stopSegmentCollection() + clock.cleanup() + }) + }) + + it('drops the buffer when the session expires without ever reporting an error', () => { + addRecord(RECORD) + // the session is gone, so nothing answers for these records any more + withholdingSessionId = undefined + releasedSessionId = undefined + + stopSegmentCollection() + worker.processAllMessages() + + expect(httpRequestSpy.send).not.toHaveBeenCalled() + expect(httpRequestSpy.sendOnExit).not.toHaveBeenCalled() + }) + + it('drops the buffer when the session is renewed into a different one', () => { + addRecord(RECORD) + withholdingSessionId = undefined + releasedSessionId = 'a-different-session' + + stopSegmentCollection() + worker.processAllMessages() + + expect(httpRequestSpy.send).not.toHaveBeenCalled() + expect(httpRequestSpy.sendOnExit).not.toHaveBeenCalled() + }) + + it('sends the buffer when its own session reports the error', () => { + addRecord(RECORD) + releasedSessionId = withholdingSessionId + withholdingSessionId = undefined + + stopSegmentCollection() + worker.processAllMessages() + + expect(httpRequestSpy.send).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/rum/src/domain/segmentCollection/segmentCollection.ts b/packages/rum/src/domain/segmentCollection/segmentCollection.ts index ac2dbfabb9..582b438fb0 100644 --- a/packages/rum/src/domain/segmentCollection/segmentCollection.ts +++ b/packages/rum/src/domain/segmentCollection/segmentCollection.ts @@ -1,13 +1,23 @@ -import type { DeflateEncoder, HttpRequest, TimeoutId } from '@flashcatcloud/browser-core' -import { isPageExitReason, ONE_SECOND, clearTimeout, setTimeout } from '@flashcatcloud/browser-core' +import type { DeflateEncoder, HttpRequest, RelativeTime, TimeoutId } from '@flashcatcloud/browser-core' +import { + addTelemetryDebug, + isPageExitReason, + ONE_SECOND, + clearTimeout, + relativeNow, + setTimeout, +} from '@flashcatcloud/browser-core' import type { LifeCycle, ViewHistory, RumSessionManager, RumConfiguration } from '@flashcatcloud/browser-rum-core' -import { LifeCycleEventType } from '@flashcatcloud/browser-rum-core' +import { LifeCycleEventType, WITHHELD_BUFFER_DURATION } from '@flashcatcloud/browser-rum-core' import type { BrowserRecord, CreationReason, SegmentContext } from '../../types' +import { RecordType } from '../../types' +import { discardSegmentData, removeSegment } from '../replayStats' import { buildReplayPayload } from './buildReplayPayload' import type { FlushReason, Segment } from './segment' import { createSegment } from './segment' export const SEGMENT_DURATION_LIMIT = 5 * ONE_SECOND + /** * beacon payload max queue size implementation is 64kb * ensure that we leave room for logs, rum and potential other users @@ -39,19 +49,43 @@ export let SEGMENT_BYTES_LIMIT = 60_000 // To help investigate session replays issues, each segment is created with a "creation reason", // indicating why the session has been created. +/** + * Lets a session record without uploading anything until it reports an error. Sessions drawn by + * `sessionReplayOnError` record from the start, but every segment is withheld: dropped on + * checkout while no error has happened, sent normally from the moment one has. + */ +export interface SegmentBuffering { + /** + * The id of the current session if it is withholding its replay, `undefined` otherwise. A segment + * remembers this at creation, so that what happens to it later is decided by the session that + * actually produced its records. + */ + getWithholdingSessionId: () => string | undefined + /** + * Whether that same session has since reported its error. Anything else — the session expired, or + * was renewed into a different one — means the records were never released and must be dropped: + * uploading them would bill a session for a replay nobody asked for and nobody can explain. + */ + isReleased: (sessionId: string) => boolean + /** Restarts the buffer from a fresh full snapshot, after the previous one was dropped. */ + restartFromFullSnapshot: () => void +} + export function startSegmentCollection( lifeCycle: LifeCycle, configuration: RumConfiguration, sessionManager: RumSessionManager, viewHistory: ViewHistory, httpRequest: HttpRequest, - encoder: DeflateEncoder + encoder: DeflateEncoder, + buffering: SegmentBuffering ) { return doStartSegmentCollection( lifeCycle, () => computeSegmentContext(configuration.applicationId, sessionManager, viewHistory), httpRequest, - encoder + encoder, + buffering ) } @@ -69,30 +103,86 @@ type SegmentCollectionState = status: SegmentCollectionStatus.SegmentPending segment: Segment expirationTimeoutId: TimeoutId + /** Only armed while the segment is withheld: bounds how much history the buffer may span. */ + bufferCheckoutTimeoutId: TimeoutId | undefined + /** Set when the segment was created while its session was withholding its replay. */ + withheldForSessionId: string | undefined } | { status: SegmentCollectionStatus.Stopped } +/** + * These two are internal and never reach the intake, so they are mapped back to a schema value where + * the next segment records why it was created. `buffer_checkout` drops a withheld buffer that has + * grown past {@link WITHHELD_BUFFER_DURATION}; `page_reactivated` cuts a segment when the page is + * switched back to, so the next one starts from the fresh full snapshot taken on the same event. + */ +type InternalFlushReason = FlushReason | 'buffer_checkout' | 'page_reactivated' + +// Recordings can stop and restart while the same encoder is still finishing a segment. +// Serialize at the encoder boundary so their metadata and index reservations cannot overlap. +let encodingQueues: WeakMap void> }> | undefined + export function doStartSegmentCollection( lifeCycle: LifeCycle, getSegmentContext: () => SegmentContext | undefined, httpRequest: HttpRequest, - encoder: DeflateEncoder + encoder: DeflateEncoder, + buffering: SegmentBuffering ) { let state: SegmentCollectionState = { status: SegmentCollectionStatus.WaitingForInitialRecord, nextSegmentCreationReason: 'init', } + // How many buffers were dropped before one was finally released. Without this, "the replay goes + // back up to a minute" is a promise nobody can check. + let droppedBufferCount = 0 + let lastBufferRestartAt: RelativeTime | undefined + let bufferRestartTimeoutId: TimeoutId | undefined + encodingQueues ||= new WeakMap() + const encodingQueue = encodingQueues.get(encoder) || { flushing: false, operations: [] } + encodingQueues.set(encoder, encodingQueue) + let stopped = false + const withholdingSessionIds = new Set() + const releasedSessionIds = new Set() + + function rememberReleases() { + withholdingSessionIds.forEach((sessionId) => { + if (buffering.isReleased(sessionId)) { + releasedSessionIds.add(sessionId) + } + }) + } + + function runWhenReady(operation: () => void) { + encodingQueue.operations.push(operation) + drainPendingOperations() + } + + function drainPendingOperations() { + while (!encodingQueue.flushing && encodingQueue.operations.length) { + encodingQueue.operations.shift()!() + } + } + + function requestFlush(reason: InternalFlushReason) { + rememberReleases() + if (reason !== 'view_change' && reason !== 'page_reactivated') { + restoreReleasedSnapshot() + } + runWhenReady(() => flushSegment(reason)) + } + const { unsubscribe: unsubscribeViewCreated } = lifeCycle.subscribe(LifeCycleEventType.VIEW_CREATED, () => { - flushSegment('view_change') + requestFlush('view_change') }) const { unsubscribe: unsubscribePageMayExit } = lifeCycle.subscribe( LifeCycleEventType.PAGE_MAY_EXIT, (pageMayExitEvent) => { - flushSegment(pageMayExitEvent.reason as FlushReason) + requestFlush(pageMayExitEvent.reason as FlushReason) } ) @@ -100,12 +190,103 @@ export function doStartSegmentCollection( // next one starts fresh with the full snapshot taken by startFullSnapshots on the same event. // Reuses the 'view_change' creation reason to avoid a schema change. const { unsubscribe: unsubscribeReactivated } = lifeCycle.subscribe(LifeCycleEventType.PAGE_REACTIVATED, () => { - flushSegment('view_change') + requestFlush('page_reactivated') }) - function flushSegment(flushReason: FlushReason) { + const { unsubscribe: unsubscribeRumEvent } = lifeCycle.subscribe( + LifeCycleEventType.RUM_EVENT_COLLECTED, + restoreReleasedSnapshot + ) + + const { unsubscribe: unsubscribeSessionReleased } = lifeCycle.subscribe( + LifeCycleEventType.SESSION_RELEASED, + ({ sessionId }) => { + if (withholdingSessionIds.has(sessionId)) { + releasedSessionIds.add(sessionId) + } + restoreReleasedSnapshot() + } + ) + + function restoreReleasedSnapshot() { + rememberReleases() + if (bufferRestartTimeoutId === undefined) { + return + } + const context = getSegmentContext() + if (context && buffering.isReleased(context.session.id)) { + // The error tracker marks the session before this listener runs. Restore the missing + // baseline now, before a view change or page exit can flush an unplayable segment. + clearTimeout(bufferRestartTimeoutId) + bufferRestartTimeoutId = undefined + lastBufferRestartAt = relativeNow() + buffering.restartFromFullSnapshot() + } else { + // The same oversized snapshot would be discarded again. Poll only for a release, without + // repeatedly serializing the document when neither an error nor new activity has arrived. + clearTimeout(bufferRestartTimeoutId) + bufferRestartTimeoutId = setTimeout(restoreReleasedSnapshot, SEGMENT_DURATION_LIMIT) + } + } + + function flushSegment(flushReason: InternalFlushReason) { + // Keep the encoder and index reservation owned by this segment until its asynchronous + // decision settles. Later records retain their emission context while waiting in FIFO order. + const withheldForSessionId = + state.status === SegmentCollectionStatus.SegmentPending ? state.withheldForSessionId : undefined + const isWithheld = withheldForSessionId !== undefined && !releasedSessionIds.has(withheldForSessionId) + if (state.status === SegmentCollectionStatus.SegmentPending) { + if (isWithheld && flushReason === 'page_reactivated') { + // The fresh full snapshot taken on the same event lands inside the withheld buffer, which + // stays replayable from it. Cutting here would only throw away what came before the switch. + return + } + + if (isWithheld && (flushReason === 'segment_duration_limit' || isPageExitReason(flushReason))) { + // Nothing can be sent while withheld, so these rotations would only throw the buffer away - + // and with it the full snapshot a released replay has to start from, leaving the rest of the + // session as incremental records nothing can be played from. A page that is merely hidden or + // frozen comes back and goes on recording; one that is really unloading takes the buffer with + // it either way. Keeping it is never worse than dropping it. + if (flushReason === 'segment_duration_limit') { + // Re-armed, so the buffer is flushed normally within one rotation of the session erroring. + // An expiring session does not lose it: the session history entry is still open when the + // recorder is stopped (`sessionManager.ts` notifies before closing it), so the stop flush + // still sees the session as released and sends. Only losing the page outright loses it. + state.expirationTimeoutId = setTimeout(() => requestFlush('segment_duration_limit'), SEGMENT_DURATION_LIMIT) + } + return + } + + encodingQueue.flushing = true state.segment.flush((metadata, encoderResult) => { + rememberReleases() + if (withheldForSessionId !== undefined && !releasedSessionIds.has(withheldForSessionId)) { + removeSegment(metadata.view.id) + // No error was reported, so this buffer is dropped rather than sent. Rolling back what its + // records contributed keeps `has_replay` and the counters on view events honest. + discardSegmentData(metadata.view.id, encoderResult.rawBytesCount, metadata.records_count) + droppedBufferCount += 1 + // Restarted from here rather than synchronously below, so the fresh full snapshot lands in + // the segment that follows this one rather than in the one being thrown away. + restartBuffer(flushReason) + encodingQueue.flushing = false + drainPendingOperations() + return + } + + if (withheldForSessionId !== undefined) { + // The first segment released by an error: report how much history it actually carried, so + // the window we promise can be compared against the one users get. + addTelemetryDebug('Error session replay buffer released', { + 'buffer.duration': metadata.end - metadata.start, + 'buffer.records_count': metadata.records_count, + 'buffer.dropped_count': droppedBufferCount, + }) + droppedBufferCount = 0 + } + const payload = buildReplayPayload(encoderResult.output, metadata, encoderResult.rawBytesCount) if (isPageExitReason(flushReason)) { @@ -113,14 +294,22 @@ export function doStartSegmentCollection( } else { httpRequest.send(payload) } + encodingQueue.flushing = false + drainPendingOperations() }) clearTimeout(state.expirationTimeoutId) + clearTimeout(state.bufferCheckoutTimeoutId) } if (flushReason !== 'stop') { state = { status: SegmentCollectionStatus.WaitingForInitialRecord, - nextSegmentCreationReason: flushReason, + nextSegmentCreationReason: + flushReason === 'buffer_checkout' + ? 'segment_duration_limit' + : flushReason === 'page_reactivated' + ? 'view_change' + : flushReason, } } else { state = { @@ -129,39 +318,103 @@ export function doStartSegmentCollection( } } - return { - addRecord: (record: BrowserRecord) => { - if (state.status === SegmentCollectionStatus.Stopped) { + /** + * A dropped buffer leaves no full snapshot behind, so the next one would not be replayable on its + * own. A view change does not need this: the new view emits its own full snapshot. + */ + function restartBuffer(flushReason: InternalFlushReason) { + if (flushReason !== 'buffer_checkout' && flushReason !== 'segment_bytes_limit') { + return + } + if (stopped || state.status === SegmentCollectionStatus.Stopped) { + // The flush that got here waited on the deflate worker, and recording was stopped in the + // meantime. Re-serializing the document now would cost a full snapshot on a page that asked + // to stop, and count records into the replay stats that no segment will ever hold. + return + } + // A snapshot can itself exceed the budget. After a rapid second discard, wait for release + // before replacing it: ordinary flushes no longer restart buffers once the session errors. + clearTimeout(bufferRestartTimeoutId) + bufferRestartTimeoutId = undefined + const now = relativeNow() + const delay = lastBufferRestartAt === undefined ? 0 : SEGMENT_DURATION_LIMIT - (now - lastBufferRestartAt) + if (delay > 0) { + bufferRestartTimeoutId = setTimeout(restoreReleasedSnapshot, delay) + } else { + lastBufferRestartAt = now + buffering.restartFromFullSnapshot() + } + } + + function addRecord( + record: BrowserRecord, + context: SegmentContext | undefined, + withheldForSessionId: string | undefined + ) { + if (state.status === SegmentCollectionStatus.Stopped) { + return + } + + if (record.type === RecordType.FullSnapshot) { + // A view change or page reactivation can supply the replacement before the timer does. + clearTimeout(bufferRestartTimeoutId) + bufferRestartTimeoutId = undefined + } + + if (state.status === SegmentCollectionStatus.WaitingForInitialRecord) { + if (!context) { return } - if (state.status === SegmentCollectionStatus.WaitingForInitialRecord) { - const context = getSegmentContext() - if (!context) { - return - } + state = { + status: SegmentCollectionStatus.SegmentPending, + segment: createSegment({ encoder, context, creationReason: state.nextSegmentCreationReason }), + expirationTimeoutId: setTimeout(() => { + requestFlush('segment_duration_limit') + }, SEGMENT_DURATION_LIMIT), + bufferCheckoutTimeoutId: + withheldForSessionId !== undefined + ? setTimeout(() => { + requestFlush('buffer_checkout') + }, WITHHELD_BUFFER_DURATION) + : undefined, + withheldForSessionId, + } + } - state = { - status: SegmentCollectionStatus.SegmentPending, - segment: createSegment({ encoder, context, creationReason: state.nextSegmentCreationReason }), - expirationTimeoutId: setTimeout(() => { - flushSegment('segment_duration_limit') - }, SEGMENT_DURATION_LIMIT), - } + state.segment.addRecord(record, (encodedBytesCount) => { + if (encodedBytesCount > SEGMENT_BYTES_LIMIT) { + requestFlush('segment_bytes_limit') } + }) + } - state.segment.addRecord(record, (encodedBytesCount) => { - if (encodedBytesCount > SEGMENT_BYTES_LIMIT) { - flushSegment('segment_bytes_limit') - } - }) + return { + addRecord: (record: BrowserRecord) => { + if (stopped) { + return + } + const context = getSegmentContext() + const withheldForSessionId = buffering.getWithholdingSessionId() + if (withheldForSessionId !== undefined) { + withholdingSessionIds.add(withheldForSessionId) + } + rememberReleases() + runWhenReady(() => addRecord(record, context, withheldForSessionId)) }, - stop: () => { - flushSegment('stop') + if (stopped) { + return + } + requestFlush('stop') + stopped = true + clearTimeout(bufferRestartTimeoutId) + bufferRestartTimeoutId = undefined unsubscribeViewCreated() unsubscribePageMayExit() unsubscribeReactivated() + unsubscribeRumEvent() + unsubscribeSessionReleased() }, } }