diff --git a/backend/services/shared/__tests__/circuitBreaker.test.ts b/backend/services/shared/__tests__/circuitBreaker.test.ts new file mode 100644 index 00000000..5edb9134 --- /dev/null +++ b/backend/services/shared/__tests__/circuitBreaker.test.ts @@ -0,0 +1,102 @@ +import { CircuitBreaker, CircuitOpenError } from '../circuitBreaker'; + +describe('CircuitBreaker (Backend)', () => { + beforeAll(() => { + jest.useFakeTimers(); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + it('should start in closed state and allow calls', async () => { + const cb = new CircuitBreaker(); + expect(cb.state).toBe('closed'); + + const result = await cb.execute(async () => 'success'); + expect(result).toBe('success'); + }); + + it('should trip to open state after consecutive failures', async () => { + const cb = new CircuitBreaker({ failureThreshold: 3 }); + + // Fail 3 times + for (let i = 0; i < 3; i++) { + await expect(cb.execute(async () => { + throw new Error('fail'); + })).rejects.toThrow('fail'); + } + + expect(cb.state).toBe('open'); + + // Next call should throw CircuitOpenError immediately without calling the action + const action = jest.fn(); + await expect(cb.execute(action)).rejects.toThrow(CircuitOpenError); + expect(action).not.toHaveBeenCalled(); + }); + + it('should transition to half-open after recovery timeout', async () => { + const cb = new CircuitBreaker({ failureThreshold: 1, recoveryTimeoutMs: 1000 }); + + await expect(cb.execute(async () => { + throw new Error('fail'); + })).rejects.toThrow('fail'); + + expect(cb.state).toBe('open'); + + // Advance time by 1000ms + jest.advanceTimersByTime(1000); + + // Call should now be allowed (half-open) + const result = await cb.execute(async () => 'success'); + expect(result).toBe('success'); + // But since successThreshold is 2 by default, it should still be half-open + expect(cb.state).toBe('half-open'); + + // Second success should close it + await cb.execute(async () => 'success2'); + expect(cb.state).toBe('closed'); + }); + + it('should trip back to open if a failure occurs while half-open', async () => { + const cb = new CircuitBreaker({ failureThreshold: 1, recoveryTimeoutMs: 1000 }); + + await expect(cb.execute(async () => { throw new Error('fail'); })).rejects.toThrow(); + expect(cb.state).toBe('open'); + + jest.advanceTimersByTime(1000); + + // Now in half-open state, if it fails again, it immediately trips to open + await expect(cb.execute(async () => { throw new Error('fail2'); })).rejects.toThrow('fail2'); + expect(cb.state).toBe('open'); + }); + + it('should reset consecutive failures on success', async () => { + const cb = new CircuitBreaker({ failureThreshold: 3 }); + + await expect(cb.execute(async () => { throw new Error('fail'); })).rejects.toThrow(); + await expect(cb.execute(async () => { throw new Error('fail'); })).rejects.toThrow(); + + // Success resets counter + await cb.execute(async () => 'success'); + + // It should now take 3 more failures to trip + await expect(cb.execute(async () => { throw new Error('fail'); })).rejects.toThrow(); + await expect(cb.execute(async () => { throw new Error('fail'); })).rejects.toThrow(); + expect(cb.state).toBe('closed'); + + await expect(cb.execute(async () => { throw new Error('fail'); })).rejects.toThrow(); + expect(cb.state).toBe('open'); + }); + + it('can be manually reset', async () => { + const cb = new CircuitBreaker({ failureThreshold: 1 }); + await expect(cb.execute(async () => { throw new Error('fail'); })).rejects.toThrow(); + expect(cb.state).toBe('open'); + + cb.reset(); + expect(cb.state).toBe('closed'); + const result = await cb.execute(async () => 'success'); + expect(result).toBe('success'); + }); +}); diff --git a/backend/services/shared/apiClient.ts b/backend/services/shared/apiClient.ts index 11d292f5..a3c6cca6 100644 --- a/backend/services/shared/apiClient.ts +++ b/backend/services/shared/apiClient.ts @@ -24,6 +24,7 @@ import type { } from './apiResponse'; import { API_VERSION_HEADER, REQUEST_ID_HEADER } from './apiResponse'; import { IDEMPOTENCY_KEY_HEADER, generateIdempotencyKey } from './idempotencyService'; +import { CircuitBreaker, CircuitBreakerOptions } from './circuitBreaker'; // ───────────────────────────────────────────────────────────────────────────── // Typed error @@ -62,6 +63,8 @@ export interface ApiClientOptions { defaultHeaders?: Record; /** Inject a custom fetch implementation (useful for testing). */ fetchImpl?: typeof fetch; + /** Configuration for the circuit breaker. */ + circuitBreaker?: CircuitBreakerOptions; } // ───────────────────────────────────────────────────────────────────────────── @@ -72,11 +75,13 @@ export class ApiClient { private readonly baseUrl: string; private readonly defaultHeaders: Record; private readonly fetchImpl: typeof fetch; + private readonly circuitBreaker: CircuitBreaker; constructor(options: ApiClientOptions) { this.baseUrl = options.baseUrl.replace(/\/$/, ''); this.defaultHeaders = options.defaultHeaders ?? {}; this.fetchImpl = options.fetchImpl ?? fetch; + this.circuitBreaker = new CircuitBreaker({ name: 'backend-api-client', ...options.circuitBreaker }); } // ── Core request method ────────────────────────────────────────────────── @@ -97,44 +102,64 @@ export class ApiClient { ...extraHeaders, }; - const response = await this.fetchImpl(url, { - method, - headers, - body: body !== undefined ? JSON.stringify(body) : undefined, + let errorToThrowOutside: any = null; + + const result = await this.circuitBreaker.execute(async () => { + const response = await this.fetchImpl(url, { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + }); + + const rawJson: unknown = await response.json(); + const isEnveloped = response.headers.get(API_VERSION_HEADER) !== null; + + if (!isEnveloped) { + if (!response.ok && response.status >= 500) { + throw new Error(`Legacy API error: ${response.status}`); + } + return { type: 'legacy', response, rawJson } as const; + } + + const envelope = rawJson as ApiResponse; + + if (!envelope.success) { + const errEnv = envelope as ApiErrorResponse; + const err = new ApiClientError( + errEnv.error.code, + errEnv.error.message, + response.status, + errEnv.meta.requestId, + errEnv.error.details, + ); + if (response.status >= 500) { + throw err; // Trip the circuit breaker + } else { + errorToThrowOutside = err; + return { type: 'error', envelope } as const; + } + } + + return { type: 'success', envelope } as const; }); - const rawJson: unknown = await response.json(); - - // ── Backward-compatibility: detect envelope vs. legacy response ────── - const isEnveloped = response.headers.get(API_VERSION_HEADER) !== null; + if (errorToThrowOutside) { + throw errorToThrowOutside; + } - if (!isEnveloped) { - // Legacy endpoint – wrap the raw body in a synthetic success envelope. + if (result.type === 'legacy') { return { success: true, - data: rawJson as T, + data: result.rawJson as T, meta: { timestamp: new Date().toISOString(), requestId, - apiVersion: 0, // 0 signals "legacy, no envelope" + apiVersion: 0, }, }; } - const envelope = rawJson as ApiResponse; - - if (!envelope.success) { - const errEnv = envelope as ApiErrorResponse; - throw new ApiClientError( - errEnv.error.code, - errEnv.error.message, - response.status, - errEnv.meta.requestId, - errEnv.error.details, - ); - } - - return envelope as ApiSuccessResponse; + return result.envelope as ApiSuccessResponse; } // ── Convenience methods ────────────────────────────────────────────────── diff --git a/backend/services/shared/circuitBreaker.ts b/backend/services/shared/circuitBreaker.ts new file mode 100644 index 00000000..f39f3517 --- /dev/null +++ b/backend/services/shared/circuitBreaker.ts @@ -0,0 +1,109 @@ +export type CircuitState = 'closed' | 'open' | 'half-open'; + +export interface CircuitBreakerOptions { + /** Number of consecutive failures before tripping to OPEN. Default: 5 */ + failureThreshold?: number; + /** Time (ms) to wait in OPEN before moving to HALF-OPEN. Default: 30000 */ + recoveryTimeoutMs?: number; + /** Number of successful calls in HALF-OPEN before moving to CLOSED. Default: 2 */ + successThreshold?: number; + /** A name or identifier for this circuit breaker. */ + name?: string; +} + +export class CircuitOpenError extends Error { + constructor(public readonly name: string, public readonly openUntil: number) { + super(`Circuit breaker "${name}" is OPEN. Recovery attempt allowed at ${new Date(openUntil).toISOString()}`); + this.name = 'CircuitOpenError'; + } +} + +export class CircuitBreaker { + public state: CircuitState = 'closed'; + private consecutiveFailures = 0; + private consecutiveSuccesses = 0; + private openUntil: number | null = null; + + private readonly failureThreshold: number; + private readonly recoveryTimeoutMs: number; + private readonly successThreshold: number; + public readonly name: string; + + constructor(options: CircuitBreakerOptions = {}) { + this.failureThreshold = options.failureThreshold ?? 5; + this.recoveryTimeoutMs = options.recoveryTimeoutMs ?? 30_000; + this.successThreshold = options.successThreshold ?? 2; + this.name = options.name ?? 'default'; + } + + /** + * Wraps an async action with the circuit breaker logic. + */ + async execute(action: () => Promise): Promise { + this.checkState(); + + if (this.state === 'open') { + throw new CircuitOpenError(this.name, this.openUntil!); + } + + try { + const result = await action(); + this.recordSuccess(); + return result; + } catch (error) { + this.recordFailure(); + throw error; + } + } + + private checkState(): void { + if (this.state === 'open' && this.openUntil !== null && Date.now() >= this.openUntil) { + this.transitionTo('half-open'); + } + } + + private recordSuccess(): void { + this.consecutiveFailures = 0; + + if (this.state === 'half-open') { + this.consecutiveSuccesses += 1; + if (this.consecutiveSuccesses >= this.successThreshold) { + this.transitionTo('closed'); + } + } + } + + private recordFailure(): void { + this.consecutiveSuccesses = 0; + + if (this.state === 'half-open') { + this.transitionTo('open'); + return; + } + + this.consecutiveFailures += 1; + if (this.state === 'closed' && this.consecutiveFailures >= this.failureThreshold) { + this.transitionTo('open'); + } + } + + private transitionTo(newState: CircuitState): void { + this.state = newState; + if (newState === 'open') { + this.openUntil = Date.now() + this.recoveryTimeoutMs; + this.consecutiveFailures = 0; + } else if (newState === 'half-open') { + this.consecutiveSuccesses = 0; + this.openUntil = null; + } else if (newState === 'closed') { + this.consecutiveFailures = 0; + this.consecutiveSuccesses = 0; + this.openUntil = null; + } + } + + // Allow manual resets + public reset(): void { + this.transitionTo('closed'); + } +} diff --git a/src/services/network/__tests__/circuitBreaker.test.ts b/src/services/network/__tests__/circuitBreaker.test.ts new file mode 100644 index 00000000..c4b96930 --- /dev/null +++ b/src/services/network/__tests__/circuitBreaker.test.ts @@ -0,0 +1,110 @@ +import { CircuitBreaker, CircuitOpenError } from '../circuitBreaker'; +import { mobileTracer, MobileTracer } from '../trace'; + +describe('CircuitBreaker (Frontend)', () => { + let mockTracer: jest.Mocked; + + beforeAll(() => { + jest.useFakeTimers(); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + beforeEach(() => { + mockTracer = { + startClientSpan: jest.fn().mockReturnValue({ context: { traceId: 'test-trace' } }), + endSpan: jest.fn(), + } as any; + }); + + it('should start in closed state and allow calls', async () => { + const cb = new CircuitBreaker({ tracer: mockTracer }); + expect(cb.state).toBe('closed'); + + const result = await cb.execute(async () => 'success'); + expect(result).toBe('success'); + + // No state change, no trace for state change + expect(mockTracer.startClientSpan).not.toHaveBeenCalled(); + }); + + it('should trip to open state after consecutive failures and trace transition', async () => { + const cb = new CircuitBreaker({ failureThreshold: 3, tracer: mockTracer }); + + // Fail 3 times + for (let i = 0; i < 3; i++) { + await expect(cb.execute(async () => { + throw new Error('fail'); + })).rejects.toThrow('fail'); + } + + expect(cb.state).toBe('open'); + expect(mockTracer.startClientSpan).toHaveBeenCalledWith('CircuitBreaker default state change', { + 'circuit.previous_state': 'closed', + 'circuit.new_state': 'open', + }); + + // Next call should throw CircuitOpenError immediately without calling the action + const action = jest.fn(); + mockTracer.startClientSpan.mockClear(); + mockTracer.endSpan.mockClear(); + + await expect(cb.execute(action)).rejects.toThrow(CircuitOpenError); + expect(action).not.toHaveBeenCalled(); + + // Fast-fail should be traced + expect(mockTracer.startClientSpan).toHaveBeenCalledWith('CircuitBreaker default fast-fail', { + 'circuit.state': 'open', + }); + expect(mockTracer.endSpan).toHaveBeenCalled(); + }); + + it('should transition to half-open after recovery timeout', async () => { + const cb = new CircuitBreaker({ failureThreshold: 1, recoveryTimeoutMs: 1000, tracer: mockTracer }); + + await expect(cb.execute(async () => { + throw new Error('fail'); + })).rejects.toThrow('fail'); + + expect(cb.state).toBe('open'); + + // Advance time by 1000ms + jest.advanceTimersByTime(1000); + + // Call should now be allowed (half-open) + const result = await cb.execute(async () => 'success'); + expect(result).toBe('success'); + // But since successThreshold is 2 by default, it should still be half-open + expect(cb.state).toBe('half-open'); + + // Second success should close it + await cb.execute(async () => 'success2'); + expect(cb.state).toBe('closed'); + }); + + it('should trip back to open if a failure occurs while half-open', async () => { + const cb = new CircuitBreaker({ failureThreshold: 1, recoveryTimeoutMs: 1000, tracer: mockTracer }); + + await expect(cb.execute(async () => { throw new Error('fail'); })).rejects.toThrow(); + expect(cb.state).toBe('open'); + + jest.advanceTimersByTime(1000); + + // Now in half-open state, if it fails again, it immediately trips to open + await expect(cb.execute(async () => { throw new Error('fail2'); })).rejects.toThrow('fail2'); + expect(cb.state).toBe('open'); + }); + + it('can be manually reset', async () => { + const cb = new CircuitBreaker({ failureThreshold: 1, tracer: mockTracer }); + await expect(cb.execute(async () => { throw new Error('fail'); })).rejects.toThrow(); + expect(cb.state).toBe('open'); + + cb.reset(); + expect(cb.state).toBe('closed'); + const result = await cb.execute(async () => 'success'); + expect(result).toBe('success'); + }); +}); diff --git a/src/services/network/apiClient.ts b/src/services/network/apiClient.ts index dae62cb0..de938a00 100644 --- a/src/services/network/apiClient.ts +++ b/src/services/network/apiClient.ts @@ -9,6 +9,7 @@ */ import { formatTraceparent, mobileTracer, MobileTracer } from './trace'; +import { CircuitBreaker, CircuitBreakerOptions } from './circuitBreaker'; export interface ApiClientOptions { baseUrl?: string; @@ -16,6 +17,8 @@ export interface ApiClientOptions { fetchImpl?: typeof fetch; /** Default headers merged into every request (e.g. content-type). */ defaultHeaders?: Record; + /** Optional circuit breaker configuration */ + circuitBreaker?: CircuitBreakerOptions; } export interface ApiRequestOptions { @@ -38,6 +41,7 @@ export class ApiClient { private readonly tracer: MobileTracer; private readonly fetchImpl: typeof fetch; private readonly defaultHeaders: Record; + private readonly circuitBreaker: CircuitBreaker; constructor(options: ApiClientOptions = {}) { this.baseUrl = (options.baseUrl ?? process.env.EXPO_PUBLIC_API_BASE_URL ?? '').replace( @@ -47,6 +51,7 @@ export class ApiClient { this.tracer = options.tracer ?? mobileTracer; this.fetchImpl = options.fetchImpl ?? fetch; this.defaultHeaders = { 'Content-Type': 'application/json', ...options.defaultHeaders }; + this.circuitBreaker = new CircuitBreaker({ name: 'frontend-api-client', tracer: this.tracer, ...options.circuitBreaker }); } async request(path: string, options: ApiRequestOptions = {}): Promise> { @@ -65,11 +70,26 @@ export class ApiClient { }; try { - const response = await this.fetchImpl(url, { - method, - headers, - body: options.body === undefined ? undefined : JSON.stringify(options.body), - }); + let response: Response; + try { + response = await this.circuitBreaker.execute(async () => { + const res = await this.fetchImpl(url, { + method, + headers, + body: options.body === undefined ? undefined : JSON.stringify(options.body), + }); + if (res.status >= 500) { + throw res; + } + return res; + }); + } catch (err) { + if (err instanceof Response) { + response = err; + } else { + throw err; + } + } const text = await response.text(); const data = (text ? JSON.parse(text) : null) as T; diff --git a/src/services/network/circuitBreaker.ts b/src/services/network/circuitBreaker.ts new file mode 100644 index 00000000..b7b1ec02 --- /dev/null +++ b/src/services/network/circuitBreaker.ts @@ -0,0 +1,131 @@ +import { mobileTracer, MobileTracer } from './trace'; + +export type CircuitState = 'closed' | 'open' | 'half-open'; + +export interface CircuitBreakerOptions { + /** Number of consecutive failures before tripping to OPEN. Default: 5 */ + failureThreshold?: number; + /** Time (ms) to wait in OPEN before moving to HALF-OPEN. Default: 30000 */ + recoveryTimeoutMs?: number; + /** Number of successful calls in HALF-OPEN before moving to CLOSED. Default: 2 */ + successThreshold?: number; + /** A name or identifier for this circuit breaker. */ + name?: string; + /** Optional tracer for telemetry */ + tracer?: MobileTracer; +} + +export class CircuitOpenError extends Error { + constructor(public readonly name: string, public readonly openUntil: number) { + super(`Circuit breaker "${name}" is OPEN. Recovery attempt allowed at ${new Date(openUntil).toISOString()}`); + this.name = 'CircuitOpenError'; + } +} + +export class CircuitBreaker { + public state: CircuitState = 'closed'; + private consecutiveFailures = 0; + private consecutiveSuccesses = 0; + private openUntil: number | null = null; + + private readonly failureThreshold: number; + private readonly recoveryTimeoutMs: number; + private readonly successThreshold: number; + public readonly name: string; + private readonly tracer: MobileTracer; + + constructor(options: CircuitBreakerOptions = {}) { + this.failureThreshold = options.failureThreshold ?? 5; + this.recoveryTimeoutMs = options.recoveryTimeoutMs ?? 30_000; + this.successThreshold = options.successThreshold ?? 2; + this.name = options.name ?? 'default'; + this.tracer = options.tracer ?? mobileTracer; + } + + /** + * Wraps an async action with the circuit breaker logic. + */ + async execute(action: () => Promise): Promise { + this.checkState(); + + if (this.state === 'open') { + const error = new CircuitOpenError(this.name, this.openUntil!); + + // Optionally trace the fast-failure + const span = this.tracer.startClientSpan(`CircuitBreaker ${this.name} fast-fail`, { + 'circuit.state': 'open', + }); + this.tracer.endSpan(span, 'error', { 'error.message': error.message }); + + throw error; + } + + try { + const result = await action(); + this.recordSuccess(); + return result; + } catch (error) { + this.recordFailure(error); + throw error; + } + } + + private checkState(): void { + if (this.state === 'open' && this.openUntil !== null && Date.now() >= this.openUntil) { + this.transitionTo('half-open'); + } + } + + private recordSuccess(): void { + this.consecutiveFailures = 0; + + if (this.state === 'half-open') { + this.consecutiveSuccesses += 1; + if (this.consecutiveSuccesses >= this.successThreshold) { + this.transitionTo('closed'); + } + } + } + + private recordFailure(error: unknown): void { + this.consecutiveSuccesses = 0; + + if (this.state === 'half-open') { + this.transitionTo('open'); + return; + } + + this.consecutiveFailures += 1; + if (this.state === 'closed' && this.consecutiveFailures >= this.failureThreshold) { + this.transitionTo('open'); + } + } + + private transitionTo(newState: CircuitState): void { + const previous = this.state; + this.state = newState; + + if (newState === 'open') { + this.openUntil = Date.now() + this.recoveryTimeoutMs; + this.consecutiveFailures = 0; + } else if (newState === 'half-open') { + this.consecutiveSuccesses = 0; + this.openUntil = null; + } else if (newState === 'closed') { + this.consecutiveFailures = 0; + this.consecutiveSuccesses = 0; + this.openUntil = null; + } + + const span = this.tracer.startClientSpan(`CircuitBreaker ${this.name} state change`, { + 'circuit.previous_state': previous, + 'circuit.new_state': newState, + }); + this.tracer.endSpan(span, 'ok'); + } + + // Allow manual resets + public reset(): void { + this.transitionTo('closed'); + } +}