diff --git a/packages/core/src/__tests__/config-validation.test.ts b/packages/core/src/__tests__/config-validation.test.ts new file mode 100644 index 000000000..17488e753 --- /dev/null +++ b/packages/core/src/__tests__/config-validation.test.ts @@ -0,0 +1,207 @@ +import { + validateRateLimitConfig, + validateBackoffConfig, +} from '../config-validation'; +import type { RateLimitConfig, BackoffConfig } from '../types'; +import { getMockLogger } from '../test-helpers'; + +describe('config-validation', () => { + let mockLogger: ReturnType; + + beforeEach(() => { + mockLogger = getMockLogger(); + }); + + describe('validateRateLimitConfig', () => { + const validConfig: RateLimitConfig = { + enabled: true, + maxRetryCount: 50, + maxRetryInterval: 300, + maxRateLimitDuration: 43200, + }; + + it('passes through valid config unchanged', () => { + const result = validateRateLimitConfig(validConfig, mockLogger); + expect(result).toEqual(validConfig); + expect(mockLogger.warn).not.toHaveBeenCalled(); + }); + + it('clamps maxRetryInterval below minimum', () => { + const result = validateRateLimitConfig( + { ...validConfig, maxRetryInterval: 0.01 }, + mockLogger + ); + expect(result.maxRetryInterval).toBe(0.1); + expect(mockLogger.warn).toHaveBeenCalled(); + }); + + it('clamps maxRetryInterval above maximum', () => { + const result = validateRateLimitConfig( + { ...validConfig, maxRetryInterval: 100000 }, + mockLogger + ); + expect(result.maxRetryInterval).toBe(86400); + }); + + it('clamps maxRateLimitDuration below absolute minimum', () => { + // With maxRetryInterval=1, 2x=2, absolute min=60 wins + const result = validateRateLimitConfig( + { ...validConfig, maxRetryInterval: 1, maxRateLimitDuration: 10 }, + mockLogger + ); + expect(result.maxRateLimitDuration).toBe(60); + }); + + it('clamps maxRetryCount to range [1, 100]', () => { + expect( + validateRateLimitConfig( + { ...validConfig, maxRetryCount: 0 }, + mockLogger + ).maxRetryCount + ).toBe(1); + expect( + validateRateLimitConfig( + { ...validConfig, maxRetryCount: 200 }, + mockLogger + ).maxRetryCount + ).toBe(100); + }); + + it('clamps maxRateLimitDuration to >= 2x maxRetryInterval', () => { + // maxRetryInterval=300, so maxRateLimitDuration must be >= 600 + const result = validateRateLimitConfig( + { ...validConfig, maxRetryInterval: 300, maxRateLimitDuration: 100 }, + mockLogger + ); + expect(result.maxRateLimitDuration).toBe(600); + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.stringContaining('2x maxRetryInterval') + ); + }); + + it('does not clamp maxRateLimitDuration when already >= 2x maxRetryInterval', () => { + const result = validateRateLimitConfig( + { ...validConfig, maxRetryInterval: 100, maxRateLimitDuration: 500 }, + mockLogger + ); + expect(result.maxRateLimitDuration).toBe(500); + }); + }); + + describe('validateBackoffConfig', () => { + const validConfig: BackoffConfig = { + enabled: true, + maxRetryCount: 50, + baseBackoffInterval: 0.5, + maxBackoffInterval: 300, + maxTotalBackoffDuration: 43200, + jitterPercent: 10, + default4xxBehavior: 'drop', + default5xxBehavior: 'retry', + statusCodeOverrides: {}, + }; + + it('passes through valid config unchanged', () => { + const result = validateBackoffConfig(validConfig, mockLogger); + expect(result).toEqual(validConfig); + expect(mockLogger.warn).not.toHaveBeenCalled(); + }); + + it('clamps maxBackoffInterval to range [0.1, 86400]', () => { + expect( + validateBackoffConfig( + { ...validConfig, maxBackoffInterval: 0.01 }, + mockLogger + ).maxBackoffInterval + ).toBe(0.1); + expect( + validateBackoffConfig( + { ...validConfig, maxBackoffInterval: 100000 }, + mockLogger + ).maxBackoffInterval + ).toBe(86400); + }); + + it('clamps baseBackoffInterval to range [0.1, 300]', () => { + expect( + validateBackoffConfig( + { ...validConfig, baseBackoffInterval: 0.01 }, + mockLogger + ).baseBackoffInterval + ).toBe(0.1); + expect( + validateBackoffConfig( + { ...validConfig, baseBackoffInterval: 500 }, + mockLogger + ).baseBackoffInterval + ).toBe(300); + }); + + it('clamps maxTotalBackoffDuration to range [60, 604800]', () => { + expect( + validateBackoffConfig( + { ...validConfig, maxTotalBackoffDuration: 10 }, + mockLogger + ).maxTotalBackoffDuration + ).toBe(600); // Gets clamped to 60 first, then to 2x maxBackoffInterval (600) + expect( + validateBackoffConfig( + { ...validConfig, maxTotalBackoffDuration: 700000 }, + mockLogger + ).maxTotalBackoffDuration + ).toBe(604800); + }); + + it('clamps jitterPercent to range [0, 100]', () => { + expect( + validateBackoffConfig({ ...validConfig, jitterPercent: -5 }, mockLogger) + .jitterPercent + ).toBe(0); + expect( + validateBackoffConfig( + { ...validConfig, jitterPercent: 150 }, + mockLogger + ).jitterPercent + ).toBe(100); + }); + + it('clamps baseBackoffInterval to <= maxBackoffInterval', () => { + const result = validateBackoffConfig( + { ...validConfig, baseBackoffInterval: 100, maxBackoffInterval: 50 }, + mockLogger + ); + expect(result.baseBackoffInterval).toBe(50); + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.stringContaining('clamped to maxBackoffInterval') + ); + }); + + it('clamps maxTotalBackoffDuration to >= 2x maxBackoffInterval', () => { + // maxBackoffInterval=300, so maxTotalBackoffDuration must be >= 600 + const result = validateBackoffConfig( + { + ...validConfig, + maxBackoffInterval: 300, + maxTotalBackoffDuration: 100, + }, + mockLogger + ); + expect(result.maxTotalBackoffDuration).toBe(600); + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.stringContaining('2x maxBackoffInterval') + ); + }); + + it('does not clamp maxTotalBackoffDuration when already >= 2x maxBackoffInterval', () => { + const result = validateBackoffConfig( + { + ...validConfig, + maxBackoffInterval: 100, + maxTotalBackoffDuration: 500, + }, + mockLogger + ); + expect(result.maxTotalBackoffDuration).toBe(500); + }); + }); +}); diff --git a/packages/core/src/__tests__/internal/fetchSettings.test.ts b/packages/core/src/__tests__/internal/fetchSettings.test.ts index b6ec2a750..f061afe5a 100644 --- a/packages/core/src/__tests__/internal/fetchSettings.test.ts +++ b/packages/core/src/__tests__/internal/fetchSettings.test.ts @@ -435,4 +435,124 @@ describe('internal #getSettings', () => { expect(spy).toHaveBeenCalled(); }); }); + + describe('httpConfig extraction', () => { + it('extracts httpConfig from CDN response and merges with defaults', async () => { + const serverHttpConfig = { + rateLimitConfig: { + enabled: true, + maxRetryCount: 50, + maxRetryInterval: 120, + maxRateLimitDuration: 3600, + }, + backoffConfig: { + enabled: true, + maxRetryCount: 50, + baseBackoffInterval: 1, + maxBackoffInterval: 120, + maxTotalBackoffDuration: 3600, + jitterPercent: 20, + default4xxBehavior: 'drop' as const, + default5xxBehavior: 'retry' as const, + statusCodeOverrides: {}, + }, + }; + + (fetch as jest.MockedFunction).mockResolvedValueOnce({ + ok: true, + json: () => + Promise.resolve({ + ...defaultIntegrationSettings, + httpConfig: serverHttpConfig, + }), + status: 200, + } as Response); + + const anotherClient = new SegmentClient({ + ...clientArgs, + logger: getMockLogger(), + }); + + await anotherClient.fetchSettings(); + const result = anotherClient.getHttpConfig(); + + expect(result).toBeDefined(); + expect(result?.rateLimitConfig?.maxRetryCount).toBe(50); + expect(result?.rateLimitConfig?.maxRetryInterval).toBe(120); + expect(result?.backoffConfig?.maxRetryCount).toBe(50); + expect(result?.backoffConfig?.jitterPercent).toBe(20); + }); + + it('returns undefined httpConfig when CDN has no httpConfig', async () => { + (fetch as jest.MockedFunction).mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve(defaultIntegrationSettings), + status: 200, + } as Response); + + const anotherClient = new SegmentClient({ + ...clientArgs, + logger: getMockLogger(), + }); + + await anotherClient.fetchSettings(); + expect(anotherClient.getHttpConfig()).toBeUndefined(); + }); + + it('returns undefined httpConfig when fetch fails', async () => { + (fetch as jest.MockedFunction).mockRejectedValueOnce( + new Error('Network error') + ); + + const anotherClient = new SegmentClient({ + ...clientArgs, + logger: getMockLogger(), + }); + + await anotherClient.fetchSettings(); + expect(anotherClient.getHttpConfig()).toBeUndefined(); + }); + + it('merges partial backoffConfig with defaults', async () => { + const partialHttpConfig = { + backoffConfig: { + enabled: true, + maxRetryCount: 25, + baseBackoffInterval: 2, + maxBackoffInterval: 60, + maxTotalBackoffDuration: 1800, + jitterPercent: 5, + default4xxBehavior: 'drop' as const, + default5xxBehavior: 'retry' as const, + statusCodeOverrides: { '501': 'drop' as const }, + }, + }; + + (fetch as jest.MockedFunction).mockResolvedValueOnce({ + ok: true, + json: () => + Promise.resolve({ + ...defaultIntegrationSettings, + httpConfig: partialHttpConfig, + }), + status: 200, + } as Response); + + const anotherClient = new SegmentClient({ + ...clientArgs, + logger: getMockLogger(), + }); + + await anotherClient.fetchSettings(); + const result = anotherClient.getHttpConfig(); + + expect(result).toBeDefined(); + // rateLimitConfig should be defaults (no server override) + expect(result?.rateLimitConfig?.enabled).toBe(true); + expect(result?.rateLimitConfig?.maxRetryCount).toBe(100); + // backoffConfig should be merged + expect(result?.backoffConfig?.maxRetryCount).toBe(25); + expect(result?.backoffConfig?.baseBackoffInterval).toBe(2); + }); + }); }); diff --git a/packages/core/src/analytics.ts b/packages/core/src/analytics.ts index 0b496ca3f..b926c3b75 100644 --- a/packages/core/src/analytics.ts +++ b/packages/core/src/analytics.ts @@ -10,6 +10,7 @@ import { workspaceDestinationFilterKey, defaultFlushInterval, defaultFlushAt, + defaultHttpConfig, maxPendingEvents, } from './constants'; import { getContext } from './context'; @@ -49,6 +50,7 @@ import { Context, DeepPartial, GroupTraits, + HttpConfig, IntegrationSettings, JsonMap, LoggerType, @@ -72,6 +74,10 @@ import { SegmentError, translateHTTPError, } from './errors'; +import { + validateRateLimitConfig, + validateBackoffConfig, +} from './config-validation'; import { QueueFlushingPlugin } from './plugins/QueueFlushingPlugin'; import { WaitingPlugin } from './plugin'; @@ -81,6 +87,9 @@ export class SegmentClient { // the config parameters for the client - a merge of user provided and default options private config: Config; + // Server-side httpConfig from CDN (undefined until fetchSettings completes) + private httpConfig?: HttpConfig; + // Storage private store: Storage; @@ -192,6 +201,14 @@ export class SegmentClient { return { ...this.config }; } + /** + * Retrieves the server-side httpConfig from CDN settings. + * Returns undefined if the CDN did not provide httpConfig (retry features disabled). + */ + getHttpConfig(): HttpConfig | undefined { + return this.httpConfig; + } + constructor({ config, logger, @@ -394,6 +411,33 @@ export class SegmentClient { const filters = this.generateFiltersMap( resJson.middlewareSettings?.routingRules ?? [] ); + + // Extract httpConfig from CDN, merge with defaults, validate and clamp + if (resJson.httpConfig) { + const mergedRateLimit = resJson.httpConfig.rateLimitConfig + ? { + ...defaultHttpConfig.rateLimitConfig!, + ...resJson.httpConfig.rateLimitConfig, + } + : defaultHttpConfig.rateLimitConfig!; + + const mergedBackoff = resJson.httpConfig.backoffConfig + ? { + ...defaultHttpConfig.backoffConfig!, + ...resJson.httpConfig.backoffConfig, + } + : defaultHttpConfig.backoffConfig!; + + this.httpConfig = { + rateLimitConfig: validateRateLimitConfig( + mergedRateLimit, + this.logger + ), + backoffConfig: validateBackoffConfig(mergedBackoff, this.logger), + }; + this.logger.info('Loaded httpConfig from CDN settings.'); + } + this.logger.info('Received settings from Segment succesfully.'); await Promise.all([ this.store.settings.set(integrations), diff --git a/packages/core/src/config-validation.ts b/packages/core/src/config-validation.ts new file mode 100644 index 000000000..acb0cf7a6 --- /dev/null +++ b/packages/core/src/config-validation.ts @@ -0,0 +1,129 @@ +import type { RateLimitConfig, BackoffConfig, LoggerType } from './types'; + +export const validateRateLimitConfig = ( + config: RateLimitConfig, + logger?: LoggerType +): RateLimitConfig => { + const validated = { ...config }; + + if (validated.maxRetryInterval < 0.1) { + logger?.warn( + `maxRetryInterval ${validated.maxRetryInterval}s clamped to 0.1s` + ); + validated.maxRetryInterval = 0.1; + } else if (validated.maxRetryInterval > 86400) { + logger?.warn( + `maxRetryInterval ${validated.maxRetryInterval}s clamped to 86400s` + ); + validated.maxRetryInterval = 86400; + } + + if (validated.maxRateLimitDuration < 60) { + logger?.warn( + `maxRateLimitDuration ${validated.maxRateLimitDuration}s clamped to 60s` + ); + validated.maxRateLimitDuration = 60; + } else if (validated.maxRateLimitDuration > 604800) { + logger?.warn( + `maxRateLimitDuration ${validated.maxRateLimitDuration}s clamped to 604800s` + ); + validated.maxRateLimitDuration = 604800; + } + + if (validated.maxRetryCount < 1) { + logger?.warn(`maxRetryCount ${validated.maxRetryCount} clamped to 1`); + validated.maxRetryCount = 1; + } else if (validated.maxRetryCount > 100) { + logger?.warn(`maxRetryCount ${validated.maxRetryCount} clamped to 100`); + validated.maxRetryCount = 100; + } + + // Relational: maxRateLimitDuration >= 2x maxRetryInterval + const minRateLimitDuration = validated.maxRetryInterval * 2; + if (validated.maxRateLimitDuration < minRateLimitDuration) { + logger?.warn( + `maxRateLimitDuration ${validated.maxRateLimitDuration}s clamped to ${minRateLimitDuration}s (2x maxRetryInterval)` + ); + validated.maxRateLimitDuration = minRateLimitDuration; + } + + return validated; +}; + +export const validateBackoffConfig = ( + config: BackoffConfig, + logger?: LoggerType +): BackoffConfig => { + const validated = { ...config }; + + if (validated.maxBackoffInterval < 0.1) { + logger?.warn( + `maxBackoffInterval ${validated.maxBackoffInterval}s clamped to 0.1s` + ); + validated.maxBackoffInterval = 0.1; + } else if (validated.maxBackoffInterval > 86400) { + logger?.warn( + `maxBackoffInterval ${validated.maxBackoffInterval}s clamped to 86400s` + ); + validated.maxBackoffInterval = 86400; + } + + if (validated.baseBackoffInterval < 0.1) { + logger?.warn( + `baseBackoffInterval ${validated.baseBackoffInterval}s clamped to 0.1s` + ); + validated.baseBackoffInterval = 0.1; + } else if (validated.baseBackoffInterval > 300) { + logger?.warn( + `baseBackoffInterval ${validated.baseBackoffInterval}s clamped to 300s` + ); + validated.baseBackoffInterval = 300; + } + + if (validated.maxTotalBackoffDuration < 60) { + logger?.warn( + `maxTotalBackoffDuration ${validated.maxTotalBackoffDuration}s clamped to 60s` + ); + validated.maxTotalBackoffDuration = 60; + } else if (validated.maxTotalBackoffDuration > 604800) { + logger?.warn( + `maxTotalBackoffDuration ${validated.maxTotalBackoffDuration}s clamped to 604800s` + ); + validated.maxTotalBackoffDuration = 604800; + } + + if (validated.jitterPercent < 0) { + logger?.warn(`jitterPercent ${validated.jitterPercent} clamped to 0`); + validated.jitterPercent = 0; + } else if (validated.jitterPercent > 100) { + logger?.warn(`jitterPercent ${validated.jitterPercent} clamped to 100`); + validated.jitterPercent = 100; + } + + if (validated.maxRetryCount < 1) { + logger?.warn(`maxRetryCount ${validated.maxRetryCount} clamped to 1`); + validated.maxRetryCount = 1; + } else if (validated.maxRetryCount > 100) { + logger?.warn(`maxRetryCount ${validated.maxRetryCount} clamped to 100`); + validated.maxRetryCount = 100; + } + + // Relational: baseBackoffInterval <= maxBackoffInterval + if (validated.baseBackoffInterval > validated.maxBackoffInterval) { + logger?.warn( + `baseBackoffInterval ${validated.baseBackoffInterval}s clamped to maxBackoffInterval ${validated.maxBackoffInterval}s` + ); + validated.baseBackoffInterval = validated.maxBackoffInterval; + } + + // Relational: maxTotalBackoffDuration >= 2x maxBackoffInterval + const minTotalDuration = validated.maxBackoffInterval * 2; + if (validated.maxTotalBackoffDuration < minTotalDuration) { + logger?.warn( + `maxTotalBackoffDuration ${validated.maxTotalBackoffDuration}s clamped to ${minTotalDuration}s (2x maxBackoffInterval)` + ); + validated.maxTotalBackoffDuration = minTotalDuration; + } + + return validated; +}; diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index cb05be4b5..3eac82418 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -1,4 +1,4 @@ -import type { Config } from './types'; +import type { Config, HttpConfig } from './types'; export const defaultApiHost = 'https://api.segment.io/v1/b'; export const settingsCDN = 'https://cdn-settings.segment.com/v1/projects'; @@ -10,6 +10,35 @@ export const defaultConfig: Config = { trackAppLifecycleEvents: false, autoAddSegmentDestination: true, useSegmentEndpoints: false, + retryStrategy: 'lazy', + autoFlushOnRetryReady: false, +}; + +export const defaultHttpConfig: HttpConfig = { + rateLimitConfig: { + enabled: true, + maxRetryCount: 100, + maxRetryInterval: 300, + maxRateLimitDuration: 43200, + }, + backoffConfig: { + enabled: true, + maxRetryCount: 100, + baseBackoffInterval: 0.5, + maxBackoffInterval: 300, + maxTotalBackoffDuration: 43200, + jitterPercent: 10, + default4xxBehavior: 'drop', + default5xxBehavior: 'retry', + statusCodeOverrides: { + '408': 'retry', + '410': 'retry', + '429': 'retry', + '460': 'retry', + '501': 'drop', + '505': 'drop', + }, + }, }; export const workspaceDestinationFilterKey = '';