From e11ed2523840d7144add08f99a47b7abb10d477b Mon Sep 17 00:00:00 2001 From: 0x860 Date: Fri, 29 May 2026 20:22:03 +0100 Subject: [PATCH 01/10] feat(scaling): add Part 37 service methods for connection breakdown and db settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #727 / #282 — query pg_stat_activity grouped by state and application name, plus a curated subset of scaling-relevant pg_settings parameters. Co-authored-by: Cursor --- backend/src/services/dbScalingService.ts | 78 ++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/backend/src/services/dbScalingService.ts b/backend/src/services/dbScalingService.ts index feb2cc8..cae57ca 100644 --- a/backend/src/services/dbScalingService.ts +++ b/backend/src/services/dbScalingService.ts @@ -201,6 +201,84 @@ export class DbScalingService { })); } + // ── Part 37 (#282) ─────────────────────────────────────────────────────── + + /** + * #282a — Connection breakdown: active connections grouped by state and + * application name from pg_stat_activity. + */ + async getConnectionBreakdown(): Promise<{ + state: string; + applicationName: string; + count: number; + }[]> { + const rows = await this.prisma.$queryRaw>` + SELECT + COALESCE(state, 'unknown') AS state, + COALESCE(application_name, '') AS application_name, + count(*) AS cnt + FROM pg_stat_activity + WHERE datname = current_database() + GROUP BY state, application_name + ORDER BY cnt DESC + `; + return rows.map(r => ({ + state: r.state, + applicationName: r.application_name, + count: Number(r.cnt), + })); + } + + /** + * #282b — Scaling-relevant database settings from pg_settings. + * Returns a curated subset of parameters that affect connection pooling, + * memory, and query performance. + */ + async getDbSettings(): Promise<{ + name: string; + setting: string; + unit: string | null; + category: string; + }[]> { + const rows = await this.prisma.$queryRaw>` + SELECT name, setting, unit, category + FROM pg_settings + WHERE name IN ( + 'max_connections', + 'shared_buffers', + 'work_mem', + 'maintenance_work_mem', + 'effective_cache_size', + 'random_page_cost', + 'seq_page_cost', + 'max_wal_size', + 'min_wal_size', + 'checkpoint_completion_target', + 'autovacuum_vacuum_scale_factor', + 'autovacuum_analyze_scale_factor', + 'statement_timeout', + 'idle_in_transaction_session_timeout', + 'lock_timeout' + ) + ORDER BY name + `; + return rows.map(r => ({ + name: r.name, + setting: r.setting, + unit: r.unit, + category: r.category, + })); + } + // ── Part 39 (#284) ─────────────────────────────────────────────────────── /** From 8339baad0fd992f10452b592fe256c7056c2c58c Mon Sep 17 00:00:00 2001 From: 0x860 Date: Fri, 29 May 2026 20:22:12 +0100 Subject: [PATCH 02/10] feat(scaling): add Part 37 controller handlers and routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #727 / #282 — expose GET /connection-breakdown and GET /db-settings with Swagger annotations following existing db-scaling conventions. Co-authored-by: Cursor --- .../src/controllers/dbScalingController.ts | 24 +++++++++++ backend/src/routes/dbScalingRoutes.ts | 41 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/backend/src/controllers/dbScalingController.ts b/backend/src/controllers/dbScalingController.ts index 912bdbe..2e5048f 100644 --- a/backend/src/controllers/dbScalingController.ts +++ b/backend/src/controllers/dbScalingController.ts @@ -104,6 +104,30 @@ export class DbScalingController { } } + // ── Part 37 (#282) ───────────────────────────────────────────────────── + + /** #282a — Connections grouped by state and application name. */ + async getConnectionBreakdown(req: Request, res: Response, next: NextFunction): Promise { + try { + const data = await service.getConnectionBreakdown(); + res.json({ success: true, data }); + } catch (err) { + logger.error({ err }, 'Failed to fetch connection breakdown'); + next(err); + } + } + + /** #282b — Scaling-relevant pg_settings parameters. */ + async getDbSettings(req: Request, res: Response, next: NextFunction): Promise { + try { + const data = await service.getDbSettings(); + res.json({ success: true, data }); + } catch (err) { + logger.error({ err }, 'Failed to fetch db settings'); + next(err); + } + } + // ── Part 39 (#284) ───────────────────────────────────────────────────── /** #284a — Lock contention between concurrent backends. */ diff --git a/backend/src/routes/dbScalingRoutes.ts b/backend/src/routes/dbScalingRoutes.ts index c29c3f4..829d3e6 100644 --- a/backend/src/routes/dbScalingRoutes.ts +++ b/backend/src/routes/dbScalingRoutes.ts @@ -106,6 +106,47 @@ router.get('/long-running-transactions', (req, res, next) => ctrl.getLongRunning // Issue #292 — vacuum / analyse stats router.get('/vacuum-stats', (req, res, next) => ctrl.getVacuumStats(req, res, next)); +// ── Part 37 (#282) ────────────────────────────────────────────────────────── + +/** + * @swagger + * /api/v1/db-scaling/connection-breakdown: + * get: + * summary: Active connections grouped by state and application (Part 37) + * description: > + * Aggregates pg_stat_activity rows for the current database by backend + * state and application_name. Useful for spotting connection leaks or + * idle-in-transaction buildup per client. + * tags: [DB Scaling] + * security: + * - bearerAuth: [] + * responses: + * 200: + * description: Connection breakdown rows + * 500: + * description: Internal server error + */ +router.get('/connection-breakdown', (req, res, next) => ctrl.getConnectionBreakdown(req, res, next)); + +/** + * @swagger + * /api/v1/db-scaling/db-settings: + * get: + * summary: Scaling-relevant PostgreSQL configuration parameters (Part 37) + * description: > + * Returns a curated subset of pg_settings that affect connection pooling, + * memory allocation, WAL sizing, autovacuum, and query timeouts. + * tags: [DB Scaling] + * security: + * - bearerAuth: [] + * responses: + * 200: + * description: Database settings list + * 500: + * description: Internal server error + */ +router.get('/db-settings', (req, res, next) => ctrl.getDbSettings(req, res, next)); + // ── Part 39 (#284) ────────────────────────────────────────────────────────── /** From 5dcd331789c744bc98521c43053238c81502cc6c Mon Sep 17 00:00:00 2001 From: 0x860 Date: Fri, 29 May 2026 20:22:23 +0100 Subject: [PATCH 03/10] test(scaling): add Part 37 integration tests for connection-breakdown and db-settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #727 / #282 — five test cases covering success, empty, and error paths. Co-authored-by: Cursor --- backend/src/__tests__/dbScalingRoutes.test.ts | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/backend/src/__tests__/dbScalingRoutes.test.ts b/backend/src/__tests__/dbScalingRoutes.test.ts index af6e647..05a4a36 100644 --- a/backend/src/__tests__/dbScalingRoutes.test.ts +++ b/backend/src/__tests__/dbScalingRoutes.test.ts @@ -33,6 +33,8 @@ const mockGetTableBloat = jest.fn(); const mockGetCacheHitRate = jest.fn(); const mockGetLongRunningTransactions = jest.fn(); const mockGetVacuumStats = jest.fn(); +const mockGetConnectionBreakdown = jest.fn(); +const mockGetDbSettings = jest.fn(); jest.mock('../services/dbScalingService.js', () => ({ DbScalingService: jest.fn().mockImplementation(() => ({ @@ -45,6 +47,8 @@ jest.mock('../services/dbScalingService.js', () => ({ getCacheHitRate: mockGetCacheHitRate, getLongRunningTransactions: mockGetLongRunningTransactions, getVacuumStats: mockGetVacuumStats, + getConnectionBreakdown: mockGetConnectionBreakdown, + getDbSettings: mockGetDbSettings, getLockContention: mockGetLockContention, getUnusedIndexes: mockGetUnusedIndexes, getReplicationLag: mockGetReplicationLag, @@ -54,6 +58,66 @@ jest.mock('../services/dbScalingService.js', () => ({ afterEach(() => jest.clearAllMocks()); +// ─── Part 37: GET /api/v1/db-scaling/connection-breakdown ──────────────────── + +describe('GET /api/v1/db-scaling/connection-breakdown', () => { + it('returns 200 with connection groups by state and application', async () => { + mockGetConnectionBreakdown.mockResolvedValue([ + { state: 'active', applicationName: 'payd-api', count: 5 }, + { state: 'idle', applicationName: 'payd-api', count: 12 }, + ]); + + const res = await request(app).get('/api/v1/db-scaling/connection-breakdown'); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.data).toHaveLength(2); + expect(res.body.data[0]).toMatchObject({ state: 'active', count: 5 }); + }); + + it('returns 200 with empty array when no connections exist', async () => { + mockGetConnectionBreakdown.mockResolvedValue([]); + + const res = await request(app).get('/api/v1/db-scaling/connection-breakdown'); + + expect(res.status).toBe(200); + expect(res.body.data).toEqual([]); + }); + + it('returns 500 when the service throws', async () => { + mockGetConnectionBreakdown.mockRejectedValue(new Error('pg error')); + + const res = await request(app).get('/api/v1/db-scaling/connection-breakdown'); + + expect(res.status).toBe(500); + }); +}); + +// ─── Part 37: GET /api/v1/db-scaling/db-settings ─────────────────────────── + +describe('GET /api/v1/db-scaling/db-settings', () => { + it('returns 200 with scaling-relevant pg_settings', async () => { + mockGetDbSettings.mockResolvedValue([ + { name: 'max_connections', setting: '100', unit: null, category: 'Connections and Authentication / Connection Settings' }, + { name: 'shared_buffers', setting: '16384', unit: '8kB', category: 'Resource Usage / Memory' }, + ]); + + const res = await request(app).get('/api/v1/db-scaling/db-settings'); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.data[0]).toMatchObject({ name: 'max_connections', setting: '100' }); + }); + + it('returns 500 when the service throws', async () => { + mockGetDbSettings.mockRejectedValue(new Error('pg error')); + + const res = await request(app).get('/api/v1/db-scaling/db-settings'); + + expect(res.status).toBe(500); + }); +}); + // ─── Part 39: GET /api/v1/db-scaling/lock-contention ───────────────────────── describe('GET /api/v1/db-scaling/lock-contention', () => { From ea10f89f1519cbaccaccbb994a03c314ee8a9add Mon Sep 17 00:00:00 2001 From: 0x860 Date: Fri, 29 May 2026 20:22:30 +0100 Subject: [PATCH 04/10] feat(scaling): add Part 38 service methods for seq-scan stats and WAL metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #728 / #283 — surface tables with high sequential scan ratios and cumulative WAL generation statistics from pg_stat_wal. Co-authored-by: Cursor --- backend/src/services/dbScalingService.ts | 85 ++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/backend/src/services/dbScalingService.ts b/backend/src/services/dbScalingService.ts index cae57ca..183609f 100644 --- a/backend/src/services/dbScalingService.ts +++ b/backend/src/services/dbScalingService.ts @@ -279,6 +279,91 @@ export class DbScalingService { })); } + // ── Part 38 (#283) ─────────────────────────────────────────────────────── + + /** + * #283a — Sequential scan stats: tables where seq_scan dominates idx_scan, + * indicating missing or unused indexes. + */ + async getSeqScanStats(limit = 20): Promise<{ + table: string; + seqScans: number; + idxScans: number; + seqTupRead: number; + idxTupFetch: number; + seqScanRatio: number; + }[]> { + const rows = await this.prisma.$queryRaw>` + SELECT relname, seq_scan, idx_scan, seq_tup_read, idx_tup_fetch + FROM pg_stat_user_tables + WHERE seq_scan > 0 + ORDER BY seq_scan DESC + LIMIT ${limit} + `; + return rows.map(r => { + const seq = Number(r.seq_scan); + const idx = Number(r.idx_scan); + return { + table: r.relname, + seqScans: seq, + idxScans: idx, + seqTupRead: Number(r.seq_tup_read), + idxTupFetch: Number(r.idx_tup_fetch), + seqScanRatio: seq + idx > 0 ? seq / (seq + idx) : 0, + }; + }); + } + + /** + * #283b — WAL generation statistics from pg_stat_wal. + * Returns cumulative WAL bytes written and record counts since last reset. + */ + async getWalStats(): Promise<{ + walRecords: number; + walFpi: number; + walBytes: number; + walBuffersFull: number; + walWrite: number; + walSync: number; + walWriteTimeMs: number; + walSyncTimeMs: number; + }> { + const rows = await this.prisma.$queryRaw>` + SELECT wal_records, wal_fpi, wal_bytes, wal_buffers_full, + wal_write, wal_sync, wal_write_time, wal_sync_time + FROM pg_stat_wal + `; + const r = rows[0] ?? { + wal_records: 0n, wal_fpi: 0n, wal_bytes: 0n, wal_buffers_full: 0n, + wal_write: 0n, wal_sync: 0n, wal_write_time: 0, wal_sync_time: 0, + }; + return { + walRecords: Number(r.wal_records), + walFpi: Number(r.wal_fpi), + walBytes: Number(r.wal_bytes), + walBuffersFull: Number(r.wal_buffers_full), + walWrite: Number(r.wal_write), + walSync: Number(r.wal_sync), + walWriteTimeMs: Math.round(r.wal_write_time / 1000), + walSyncTimeMs: Math.round(r.wal_sync_time / 1000), + }; + } + // ── Part 39 (#284) ─────────────────────────────────────────────────────── /** From e04f4c41f4c9ffb1631054b3a96b303664b63bae Mon Sep 17 00:00:00 2001 From: 0x860 Date: Fri, 29 May 2026 20:22:37 +0100 Subject: [PATCH 05/10] feat(scaling): add Part 38 controller handlers and routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #728 / #283 — expose GET /seq-scan-stats and GET /wal-stats with parameter validation and Swagger annotations. Co-authored-by: Cursor --- .../src/controllers/dbScalingController.ts | 29 +++++++++++ backend/src/routes/dbScalingRoutes.ts | 49 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/backend/src/controllers/dbScalingController.ts b/backend/src/controllers/dbScalingController.ts index 2e5048f..684ae1f 100644 --- a/backend/src/controllers/dbScalingController.ts +++ b/backend/src/controllers/dbScalingController.ts @@ -128,6 +128,35 @@ export class DbScalingController { } } + // ── Part 38 (#283) ───────────────────────────────────────────────────── + + /** #283a — Tables with high sequential scan counts. */ + async getSeqScanStats(req: Request, res: Response, next: NextFunction): Promise { + try { + const limit = Math.min(Number(req.query['limit'] ?? 20), 100); + if (isNaN(limit) || limit < 1) { + res.status(400).json({ success: false, error: 'limit must be a positive integer' }); + return; + } + const data = await service.getSeqScanStats(limit); + res.json({ success: true, data }); + } catch (err) { + logger.error({ err }, 'Failed to fetch sequential scan stats'); + next(err); + } + } + + /** #283b — WAL generation statistics. */ + async getWalStats(req: Request, res: Response, next: NextFunction): Promise { + try { + const data = await service.getWalStats(); + res.json({ success: true, data }); + } catch (err) { + logger.error({ err }, 'Failed to fetch WAL stats'); + next(err); + } + } + // ── Part 39 (#284) ───────────────────────────────────────────────────── /** #284a — Lock contention between concurrent backends. */ diff --git a/backend/src/routes/dbScalingRoutes.ts b/backend/src/routes/dbScalingRoutes.ts index 829d3e6..499b55e 100644 --- a/backend/src/routes/dbScalingRoutes.ts +++ b/backend/src/routes/dbScalingRoutes.ts @@ -147,6 +147,55 @@ router.get('/connection-breakdown', (req, res, next) => ctrl.getConnectionBreakd */ router.get('/db-settings', (req, res, next) => ctrl.getDbSettings(req, res, next)); +// ── Part 38 (#283) ────────────────────────────────────────────────────────── + +/** + * @swagger + * /api/v1/db-scaling/seq-scan-stats: + * get: + * summary: Tables with high sequential scan counts (Part 38) + * description: > + * Lists user tables ordered by seq_scan descending. A high seqScanRatio + * indicates missing or unused indexes worth investigating. + * tags: [DB Scaling] + * security: + * - bearerAuth: [] + * parameters: + * - in: query + * name: limit + * schema: + * type: integer + * default: 20 + * maximum: 100 + * responses: + * 200: + * description: Sequential scan statistics per table + * 400: + * description: Invalid limit parameter + * 500: + * description: Internal server error + */ +router.get('/seq-scan-stats', (req, res, next) => ctrl.getSeqScanStats(req, res, next)); + +/** + * @swagger + * /api/v1/db-scaling/wal-stats: + * get: + * summary: WAL generation statistics since last stats reset (Part 38) + * description: > + * Queries pg_stat_wal for cumulative WAL records, bytes written, + * buffer overflows, and write/sync timing. + * tags: [DB Scaling] + * security: + * - bearerAuth: [] + * responses: + * 200: + * description: WAL statistics snapshot + * 500: + * description: Internal server error + */ +router.get('/wal-stats', (req, res, next) => ctrl.getWalStats(req, res, next)); + // ── Part 39 (#284) ────────────────────────────────────────────────────────── /** From 5da4e1e7a092fe516069e85d9cec727a938cf528 Mon Sep 17 00:00:00 2001 From: 0x860 Date: Fri, 29 May 2026 20:22:46 +0100 Subject: [PATCH 06/10] test(scaling): add Part 38 integration tests for seq-scan-stats and wal-stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #728 / #283 — six test cases covering limit validation and error paths. Co-authored-by: Cursor --- backend/src/__tests__/dbScalingRoutes.test.ts | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/backend/src/__tests__/dbScalingRoutes.test.ts b/backend/src/__tests__/dbScalingRoutes.test.ts index 05a4a36..8fff58c 100644 --- a/backend/src/__tests__/dbScalingRoutes.test.ts +++ b/backend/src/__tests__/dbScalingRoutes.test.ts @@ -35,6 +35,8 @@ const mockGetLongRunningTransactions = jest.fn(); const mockGetVacuumStats = jest.fn(); const mockGetConnectionBreakdown = jest.fn(); const mockGetDbSettings = jest.fn(); +const mockGetSeqScanStats = jest.fn(); +const mockGetWalStats = jest.fn(); jest.mock('../services/dbScalingService.js', () => ({ DbScalingService: jest.fn().mockImplementation(() => ({ @@ -49,6 +51,8 @@ jest.mock('../services/dbScalingService.js', () => ({ getVacuumStats: mockGetVacuumStats, getConnectionBreakdown: mockGetConnectionBreakdown, getDbSettings: mockGetDbSettings, + getSeqScanStats: mockGetSeqScanStats, + getWalStats: mockGetWalStats, getLockContention: mockGetLockContention, getUnusedIndexes: mockGetUnusedIndexes, getReplicationLag: mockGetReplicationLag, @@ -118,6 +122,71 @@ describe('GET /api/v1/db-scaling/db-settings', () => { }); }); +// ─── Part 38: GET /api/v1/db-scaling/seq-scan-stats ────────────────────────── + +describe('GET /api/v1/db-scaling/seq-scan-stats', () => { + it('returns 200 with sequential scan statistics', async () => { + mockGetSeqScanStats.mockResolvedValue([ + { table: 'audit_logs', seqScans: 500, idxScans: 50, seqTupRead: 10000, idxTupFetch: 200, seqScanRatio: 0.909 }, + ]); + + const res = await request(app).get('/api/v1/db-scaling/seq-scan-stats'); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.data[0]).toMatchObject({ table: 'audit_logs', seqScans: 500 }); + }); + + it('respects the ?limit query parameter', async () => { + mockGetSeqScanStats.mockResolvedValue([]); + + await request(app).get('/api/v1/db-scaling/seq-scan-stats?limit=5'); + + expect(mockGetSeqScanStats).toHaveBeenCalledWith(5); + }); + + it('returns 400 for an invalid limit', async () => { + const res = await request(app).get('/api/v1/db-scaling/seq-scan-stats?limit=abc'); + + expect(res.status).toBe(400); + expect(res.body.success).toBe(false); + }); + + it('returns 500 when the service throws', async () => { + mockGetSeqScanStats.mockRejectedValue(new Error('pg error')); + + const res = await request(app).get('/api/v1/db-scaling/seq-scan-stats'); + + expect(res.status).toBe(500); + }); +}); + +// ─── Part 38: GET /api/v1/db-scaling/wal-stats ────────────────────────────── + +describe('GET /api/v1/db-scaling/wal-stats', () => { + it('returns 200 with WAL generation statistics', async () => { + mockGetWalStats.mockResolvedValue({ + walRecords: 1000, walFpi: 50, walBytes: 5242880, + walBuffersFull: 2, walWrite: 100, walSync: 80, + walWriteTimeMs: 150, walSyncTimeMs: 200, + }); + + const res = await request(app).get('/api/v1/db-scaling/wal-stats'); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.data).toMatchObject({ walBytes: 5242880, walRecords: 1000 }); + }); + + it('returns 500 when the service throws', async () => { + mockGetWalStats.mockRejectedValue(new Error('pg error')); + + const res = await request(app).get('/api/v1/db-scaling/wal-stats'); + + expect(res.status).toBe(500); + }); +}); + // ─── Part 39: GET /api/v1/db-scaling/lock-contention ───────────────────────── describe('GET /api/v1/db-scaling/lock-contention', () => { From 9db03dcf9962f925add72ba62a3df00f410491b2 Mon Sep 17 00:00:00 2001 From: 0x860 Date: Fri, 29 May 2026 20:23:03 +0100 Subject: [PATCH 07/10] feat(scaling): add Part 42 service methods for bgwriter and temp file usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #732 / #287 — query pg_stat_bgwriter for checkpoint/buffer stats and pg_stat_database for temporary file spill metrics on the current database. Co-authored-by: Cursor --- backend/src/services/dbScalingService.ts | 180 +++++++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/backend/src/services/dbScalingService.ts b/backend/src/services/dbScalingService.ts index 183609f..99bf8e7 100644 --- a/backend/src/services/dbScalingService.ts +++ b/backend/src/services/dbScalingService.ts @@ -364,6 +364,186 @@ export class DbScalingService { }; } + // ── Part 42 (#287) ─────────────────────────────────────────────────────── + + /** + * #287a — Background writer and checkpoint statistics from pg_stat_bgwriter. + * Surfaces checkpoint frequency, buffer writes, and allocation counts. + */ + async getBgwriterStats(): Promise<{ + checkpointsTimed: number; + checkpointsReq: number; + checkpointWriteTimeMs: number; + checkpointSyncTimeMs: number; + buffersCheckpoint: number; + buffersClean: number; + maxwrittenClean: number; + buffersBackend: number; + buffersBackendFsync: number; + buffersAlloc: number; + }> { + const rows = await this.prisma.$queryRaw>` + SELECT checkpoints_timed, checkpoints_req, + checkpoint_write_time, checkpoint_sync_time, + buffers_checkpoint, buffers_clean, maxwritten_clean, + buffers_backend, buffers_backend_fsync, buffers_alloc + FROM pg_stat_bgwriter + `; + const r = rows[0] ?? { + checkpoints_timed: 0n, checkpoints_req: 0n, checkpoint_write_time: 0, + checkpoint_sync_time: 0, buffers_checkpoint: 0n, buffers_clean: 0n, + maxwritten_clean: 0n, buffers_backend: 0n, buffers_backend_fsync: 0n, + buffers_alloc: 0n, + }; + return { + checkpointsTimed: Number(r.checkpoints_timed), + checkpointsReq: Number(r.checkpoints_req), + checkpointWriteTimeMs: Math.round(r.checkpoint_write_time), + checkpointSyncTimeMs: Math.round(r.checkpoint_sync_time), + buffersCheckpoint: Number(r.buffers_checkpoint), + buffersClean: Number(r.buffers_clean), + maxwrittenClean: Number(r.maxwritten_clean), + buffersBackend: Number(r.buffers_backend), + buffersBackendFsync: Number(r.buffers_backend_fsync), + buffersAlloc: Number(r.buffers_alloc), + }; + } + + /** + * #287b — Temporary file usage per database from pg_stat_database. + * High temp_bytes indicates queries spilling to disk due to memory pressure. + */ + async getTempFileUsage(): Promise<{ + database: string; + tempFiles: number; + tempBytes: number; + tempBytesPretty: string; + }> { + const rows = await this.prisma.$queryRaw>` + SELECT datname, temp_files, temp_bytes, + pg_size_pretty(temp_bytes) AS temp_bytes_pretty + FROM pg_stat_database + WHERE datname = current_database() + `; + const r = rows[0] ?? { datname: '', temp_files: 0n, temp_bytes: 0n, temp_bytes_pretty: '0 bytes' }; + return { + database: r.datname, + tempFiles: Number(r.temp_files), + tempBytes: Number(r.temp_bytes), + tempBytesPretty: r.temp_bytes_pretty, + }; + } + + // ── Part 50 (#295) ─────────────────────────────────────────────────────── + + /** + * #295a — Database-wide transaction and conflict statistics. + * Includes xact_commit/rollback counts, deadlocks, and temp usage for + * capacity planning and wraparound risk assessment. + */ + async getDatabaseStats(): Promise<{ + database: string; + numBackends: number; + xactCommit: number; + xactRollback: number; + blksRead: number; + blksHit: number; + cacheHitRatio: number; + deadlocks: number; + tempFiles: number; + tempBytes: number; + }> { + const rows = await this.prisma.$queryRaw>` + SELECT datname, numbackends, xact_commit, xact_rollback, + blks_read, blks_hit, deadlocks, temp_files, temp_bytes + FROM pg_stat_database + WHERE datname = current_database() + `; + const r = rows[0] ?? { + datname: '', numbackends: 0, xact_commit: 0n, xact_rollback: 0n, + blks_read: 0n, blks_hit: 0n, deadlocks: 0n, temp_files: 0n, temp_bytes: 0n, + }; + const read = Number(r.blks_read); + const hit = Number(r.blks_hit); + return { + database: r.datname, + numBackends: r.numbackends, + xactCommit: Number(r.xact_commit), + xactRollback: Number(r.xact_rollback), + blksRead: read, + blksHit: hit, + cacheHitRatio: read + hit > 0 ? hit / (read + hit) : 1, + deadlocks: Number(r.deadlocks), + tempFiles: Number(r.temp_files), + tempBytes: Number(r.temp_bytes), + }; + } + + /** + * #295b — Block I/O timing statistics from pg_stat_database. + * Surfaces cumulative read/write time for diagnosing storage bottlenecks. + */ + async getBlockIoStats(): Promise<{ + database: string; + blkReadTimeMs: number; + blkWriteTimeMs: number; + sessionTimeMs: number; + activeTimeMs: number; + idleInTransactionTimeMs: number; + }> { + const rows = await this.prisma.$queryRaw>` + SELECT datname, blk_read_time, blk_write_time, + session_time, active_time, idle_in_transaction_time + FROM pg_stat_database + WHERE datname = current_database() + `; + const r = rows[0] ?? { + datname: '', blk_read_time: 0, blk_write_time: 0, + session_time: 0, active_time: 0, idle_in_transaction_time: 0, + }; + return { + database: r.datname, + blkReadTimeMs: Math.round(r.blk_read_time), + blkWriteTimeMs: Math.round(r.blk_write_time), + sessionTimeMs: Math.round(r.session_time), + activeTimeMs: Math.round(r.active_time), + idleInTransactionTimeMs: Math.round(r.idle_in_transaction_time), + }; + } + // ── Part 39 (#284) ─────────────────────────────────────────────────────── /** From aded119caa60886be17680bb25b4881565f09d62 Mon Sep 17 00:00:00 2001 From: 0x860 Date: Fri, 29 May 2026 20:23:22 +0100 Subject: [PATCH 08/10] feat(scaling): add Part 42 controller handlers and routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #732 / #287 — expose GET /bgwriter-stats and GET /temp-file-usage with Swagger annotations following existing db-scaling conventions. Co-authored-by: Cursor --- .../src/controllers/dbScalingController.ts | 48 +++++++++++ backend/src/routes/dbScalingRoutes.ts | 80 +++++++++++++++++++ 2 files changed, 128 insertions(+) diff --git a/backend/src/controllers/dbScalingController.ts b/backend/src/controllers/dbScalingController.ts index 684ae1f..1231e8e 100644 --- a/backend/src/controllers/dbScalingController.ts +++ b/backend/src/controllers/dbScalingController.ts @@ -157,6 +157,54 @@ export class DbScalingController { } } + // ── Part 42 (#287) ───────────────────────────────────────────────────── + + /** #287a — Background writer and checkpoint statistics. */ + async getBgwriterStats(req: Request, res: Response, next: NextFunction): Promise { + try { + const data = await service.getBgwriterStats(); + res.json({ success: true, data }); + } catch (err) { + logger.error({ err }, 'Failed to fetch bgwriter stats'); + next(err); + } + } + + /** #287b — Temporary file usage for the current database. */ + async getTempFileUsage(req: Request, res: Response, next: NextFunction): Promise { + try { + const data = await service.getTempFileUsage(); + res.json({ success: true, data }); + } catch (err) { + logger.error({ err }, 'Failed to fetch temp file usage'); + next(err); + } + } + + // ── Part 50 (#295) ───────────────────────────────────────────────────── + + /** #295a — Database-wide transaction and conflict statistics. */ + async getDatabaseStats(req: Request, res: Response, next: NextFunction): Promise { + try { + const data = await service.getDatabaseStats(); + res.json({ success: true, data }); + } catch (err) { + logger.error({ err }, 'Failed to fetch database stats'); + next(err); + } + } + + /** #295b — Block I/O timing statistics. */ + async getBlockIoStats(req: Request, res: Response, next: NextFunction): Promise { + try { + const data = await service.getBlockIoStats(); + res.json({ success: true, data }); + } catch (err) { + logger.error({ err }, 'Failed to fetch block I/O stats'); + next(err); + } + } + // ── Part 39 (#284) ───────────────────────────────────────────────────── /** #284a — Lock contention between concurrent backends. */ diff --git a/backend/src/routes/dbScalingRoutes.ts b/backend/src/routes/dbScalingRoutes.ts index 499b55e..be28e7e 100644 --- a/backend/src/routes/dbScalingRoutes.ts +++ b/backend/src/routes/dbScalingRoutes.ts @@ -196,6 +196,86 @@ router.get('/seq-scan-stats', (req, res, next) => ctrl.getSeqScanStats(req, res, */ router.get('/wal-stats', (req, res, next) => ctrl.getWalStats(req, res, next)); +// ── Part 42 (#287) ────────────────────────────────────────────────────────── + +/** + * @swagger + * /api/v1/db-scaling/bgwriter-stats: + * get: + * summary: Background writer and checkpoint statistics (Part 42) + * description: > + * Queries pg_stat_bgwriter for checkpoint counts, buffer writes, + * and allocation metrics since the last stats reset. + * tags: [DB Scaling] + * security: + * - bearerAuth: [] + * responses: + * 200: + * description: Bgwriter statistics snapshot + * 500: + * description: Internal server error + */ +router.get('/bgwriter-stats', (req, res, next) => ctrl.getBgwriterStats(req, res, next)); + +/** + * @swagger + * /api/v1/db-scaling/temp-file-usage: + * get: + * summary: Temporary file spill usage for the current database (Part 42) + * description: > + * Returns temp_files and temp_bytes from pg_stat_database. High values + * indicate queries exceeding work_mem and spilling sorts/hashes to disk. + * tags: [DB Scaling] + * security: + * - bearerAuth: [] + * responses: + * 200: + * description: Temp file usage snapshot + * 500: + * description: Internal server error + */ +router.get('/temp-file-usage', (req, res, next) => ctrl.getTempFileUsage(req, res, next)); + +// ── Part 50 (#295) ────────────────────────────────────────────────────────── + +/** + * @swagger + * /api/v1/db-scaling/database-stats: + * get: + * summary: Database-wide transaction and conflict statistics (Part 50) + * description: > + * Aggregates xact_commit/rollback, block cache hit ratio, deadlocks, + * and temp usage from pg_stat_database for the current database. + * tags: [DB Scaling] + * security: + * - bearerAuth: [] + * responses: + * 200: + * description: Database statistics snapshot + * 500: + * description: Internal server error + */ +router.get('/database-stats', (req, res, next) => ctrl.getDatabaseStats(req, res, next)); + +/** + * @swagger + * /api/v1/db-scaling/block-io-stats: + * get: + * summary: Block I/O and session timing statistics (Part 50) + * description: > + * Returns cumulative blk_read_time, blk_write_time, session_time, + * active_time, and idle_in_transaction_time from pg_stat_database. + * tags: [DB Scaling] + * security: + * - bearerAuth: [] + * responses: + * 200: + * description: Block I/O timing snapshot + * 500: + * description: Internal server error + */ +router.get('/block-io-stats', (req, res, next) => ctrl.getBlockIoStats(req, res, next)); + // ── Part 39 (#284) ────────────────────────────────────────────────────────── /** From 87736bf0cc9da133ccb3cb6a19003151c76cdc31 Mon Sep 17 00:00:00 2001 From: 0x860 Date: Fri, 29 May 2026 20:23:43 +0100 Subject: [PATCH 09/10] test(scaling): add Part 42 integration tests for bgwriter-stats and temp-file-usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #732 / #287 — four test cases covering success and error paths. Co-authored-by: Cursor --- backend/src/__tests__/dbScalingRoutes.test.ts | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/backend/src/__tests__/dbScalingRoutes.test.ts b/backend/src/__tests__/dbScalingRoutes.test.ts index 8fff58c..2906835 100644 --- a/backend/src/__tests__/dbScalingRoutes.test.ts +++ b/backend/src/__tests__/dbScalingRoutes.test.ts @@ -37,6 +37,10 @@ const mockGetConnectionBreakdown = jest.fn(); const mockGetDbSettings = jest.fn(); const mockGetSeqScanStats = jest.fn(); const mockGetWalStats = jest.fn(); +const mockGetBgwriterStats = jest.fn(); +const mockGetTempFileUsage = jest.fn(); +const mockGetDatabaseStats = jest.fn(); +const mockGetBlockIoStats = jest.fn(); jest.mock('../services/dbScalingService.js', () => ({ DbScalingService: jest.fn().mockImplementation(() => ({ @@ -53,6 +57,10 @@ jest.mock('../services/dbScalingService.js', () => ({ getDbSettings: mockGetDbSettings, getSeqScanStats: mockGetSeqScanStats, getWalStats: mockGetWalStats, + getBgwriterStats: mockGetBgwriterStats, + getTempFileUsage: mockGetTempFileUsage, + getDatabaseStats: mockGetDatabaseStats, + getBlockIoStats: mockGetBlockIoStats, getLockContention: mockGetLockContention, getUnusedIndexes: mockGetUnusedIndexes, getReplicationLag: mockGetReplicationLag, @@ -187,6 +195,108 @@ describe('GET /api/v1/db-scaling/wal-stats', () => { }); }); +// ─── Part 42: GET /api/v1/db-scaling/bgwriter-stats ────────────────────────── + +describe('GET /api/v1/db-scaling/bgwriter-stats', () => { + it('returns 200 with bgwriter statistics', async () => { + mockGetBgwriterStats.mockResolvedValue({ + checkpointsTimed: 100, checkpointsReq: 5, + checkpointWriteTimeMs: 5000, checkpointSyncTimeMs: 2000, + buffersCheckpoint: 1000, buffersClean: 500, maxwrittenClean: 10, + buffersBackend: 200, buffersBackendFsync: 50, buffersAlloc: 300, + }); + + const res = await request(app).get('/api/v1/db-scaling/bgwriter-stats'); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.data).toMatchObject({ checkpointsTimed: 100, buffersCheckpoint: 1000 }); + }); + + it('returns 500 when the service throws', async () => { + mockGetBgwriterStats.mockRejectedValue(new Error('pg error')); + + const res = await request(app).get('/api/v1/db-scaling/bgwriter-stats'); + + expect(res.status).toBe(500); + }); +}); + +// ─── Part 42: GET /api/v1/db-scaling/temp-file-usage ──────────────────────── + +describe('GET /api/v1/db-scaling/temp-file-usage', () => { + it('returns 200 with temp file usage for current database', async () => { + mockGetTempFileUsage.mockResolvedValue({ + database: 'payd', tempFiles: 42, tempBytes: 1048576, tempBytesPretty: '1024 kB', + }); + + const res = await request(app).get('/api/v1/db-scaling/temp-file-usage'); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.data).toMatchObject({ tempFiles: 42, tempBytes: 1048576 }); + }); + + it('returns 500 when the service throws', async () => { + mockGetTempFileUsage.mockRejectedValue(new Error('pg error')); + + const res = await request(app).get('/api/v1/db-scaling/temp-file-usage'); + + expect(res.status).toBe(500); + }); +}); + +// ─── Part 50: GET /api/v1/db-scaling/database-stats ───────────────────────── + +describe('GET /api/v1/db-scaling/database-stats', () => { + it('returns 200 with database-wide statistics', async () => { + mockGetDatabaseStats.mockResolvedValue({ + database: 'payd', numBackends: 10, xactCommit: 50000, xactRollback: 100, + blksRead: 1000, blksHit: 99000, cacheHitRatio: 0.99, deadlocks: 0, + tempFiles: 5, tempBytes: 524288, + }); + + const res = await request(app).get('/api/v1/db-scaling/database-stats'); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.data).toMatchObject({ xactCommit: 50000, cacheHitRatio: 0.99 }); + }); + + it('returns 500 when the service throws', async () => { + mockGetDatabaseStats.mockRejectedValue(new Error('pg error')); + + const res = await request(app).get('/api/v1/db-scaling/database-stats'); + + expect(res.status).toBe(500); + }); +}); + +// ─── Part 50: GET /api/v1/db-scaling/block-io-stats ───────────────────────── + +describe('GET /api/v1/db-scaling/block-io-stats', () => { + it('returns 200 with block I/O timing statistics', async () => { + mockGetBlockIoStats.mockResolvedValue({ + database: 'payd', blkReadTimeMs: 1500, blkWriteTimeMs: 800, + sessionTimeMs: 3600000, activeTimeMs: 1800000, idleInTransactionTimeMs: 5000, + }); + + const res = await request(app).get('/api/v1/db-scaling/block-io-stats'); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.data).toMatchObject({ blkReadTimeMs: 1500, sessionTimeMs: 3600000 }); + }); + + it('returns 500 when the service throws', async () => { + mockGetBlockIoStats.mockRejectedValue(new Error('pg error')); + + const res = await request(app).get('/api/v1/db-scaling/block-io-stats'); + + expect(res.status).toBe(500); + }); +}); + // ─── Part 39: GET /api/v1/db-scaling/lock-contention ───────────────────────── describe('GET /api/v1/db-scaling/lock-contention', () => { From 03be1de1084ee39e2999b5f1b16900bb7c0aee81 Mon Sep 17 00:00:00 2001 From: 0x860 Date: Fri, 29 May 2026 20:23:47 +0100 Subject: [PATCH 10/10] test(scaling): add Part 50 integration tests for database-stats and block-io-stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #740 / #295 — four test cases covering database-wide transaction metrics and block I/O timing endpoints; update test file header for all covered parts. Co-authored-by: Cursor --- backend/src/__tests__/dbScalingRoutes.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/src/__tests__/dbScalingRoutes.test.ts b/backend/src/__tests__/dbScalingRoutes.test.ts index 2906835..ee62c84 100644 --- a/backend/src/__tests__/dbScalingRoutes.test.ts +++ b/backend/src/__tests__/dbScalingRoutes.test.ts @@ -1,8 +1,12 @@ /** - * Integration tests for the DB Scaling endpoints (Parts 39 & 40). + * Integration tests for the DB Scaling endpoints (Parts 37, 38, 39, 40, 42 & 50). * + * Issues #282 (Part 37) — connection breakdown, db settings + * Issues #283 (Part 38) — seq scan stats, WAL stats * Issues #284 (Part 39) — lock contention, unused indexes * Issues #285 (Part 40) — replication lag, table sizes + * Issues #287 (Part 42) — bgwriter stats, temp file usage + * Issues #295 (Part 50) — database stats, block I/O stats * * Strategy * ─────────