From 84c247f20cc4c39d73ef69be0ccb951e150bf16a Mon Sep 17 00:00:00 2001 From: Ammar Ansari Date: Tue, 12 Aug 2025 16:02:15 +0200 Subject: [PATCH 1/3] Call start time performance tests --- .github/workflows/browser-js-production.yml | 1 + .github/workflows/browser-js-staging.yml | 1 + internal/e2e-js/playwright.config.ts | 16 +++- .../tests/callfabric/callStartTime.spec.ts | 93 +++++++++++++++++++ .../e2e-js/tests/roomSessionStartTime.spec.ts | 50 ++++++++++ internal/e2e-js/utils.ts | 19 ++-- 6 files changed, 168 insertions(+), 12 deletions(-) create mode 100644 internal/e2e-js/tests/callfabric/callStartTime.spec.ts create mode 100644 internal/e2e-js/tests/roomSessionStartTime.spec.ts diff --git a/.github/workflows/browser-js-production.yml b/.github/workflows/browser-js-production.yml index c88c4531a..a8d7f4a86 100644 --- a/.github/workflows/browser-js-production.yml +++ b/.github/workflows/browser-js-production.yml @@ -30,6 +30,7 @@ jobs: callfabric, renegotiation, videoElement, + performance, v2WebRTC, ] steps: diff --git a/.github/workflows/browser-js-staging.yml b/.github/workflows/browser-js-staging.yml index 4f23353d9..5ced864cc 100644 --- a/.github/workflows/browser-js-staging.yml +++ b/.github/workflows/browser-js-staging.yml @@ -29,6 +29,7 @@ jobs: callfabric, renegotiation, videoElement, + performance, v2WebRTC, ] steps: diff --git a/internal/e2e-js/playwright.config.ts b/internal/e2e-js/playwright.config.ts index 6681df342..1ecb7d7ea 100644 --- a/internal/e2e-js/playwright.config.ts +++ b/internal/e2e-js/playwright.config.ts @@ -61,7 +61,11 @@ const videoElementTests = [ 'buildVideoWithVideoSDK.spec.ts', 'buildVideoWithFabricSDK.spec.ts', ] -const v2WebRTC = ['v2WebrtcFromRest.spec.ts', 'webrtcCalling.spec.ts'] +const performanceTests = [ + 'callStartTime.spec.ts', + 'roomSessionStartTime.spec.ts', +] +const v2WebRTCTests = ['v2WebrtcFromRest.spec.ts', 'webrtcCalling.spec.ts'] const useDesktopChrome: PlaywrightTestConfig['use'] = { ...devices['Desktop Chrome'], @@ -104,7 +108,8 @@ const config: PlaywrightTestConfig = { ...callfabricTests, ...renegotiationTests, ...videoElementTests, - ...v2WebRTC, + ...performanceTests, + ...v2WebRTCTests, ], }, { @@ -152,10 +157,15 @@ const config: PlaywrightTestConfig = { use: useDesktopChrome, testMatch: videoElementTests, }, + { + name: 'performance', + use: useDesktopChrome, + testMatch: performanceTests, + }, { name: 'v2WebRTC', use: useDesktopChrome, - testMatch: v2WebRTC, + testMatch: v2WebRTCTests, }, ], } diff --git a/internal/e2e-js/tests/callfabric/callStartTime.spec.ts b/internal/e2e-js/tests/callfabric/callStartTime.spec.ts new file mode 100644 index 000000000..bdfa435ca --- /dev/null +++ b/internal/e2e-js/tests/callfabric/callStartTime.spec.ts @@ -0,0 +1,93 @@ +import { uuid } from '@signalwire/core' +import { test, expect } from '../../fixtures' +import { SERVER_URL, createCFClient } from '../../utils' +import { SignalWireContract } from '@signalwire/js' + +const MAX_CALL_SETUP_TIME_MS = 5000 + +test.describe('CallFabric Start Time', () => { + test('should join a video room within 5 seconds', async ({ + createCustomPage, + resource, + }) => { + const page = await createCustomPage({ name: '[page]' }) + await page.goto(SERVER_URL) + + const roomName = `e2e_${uuid()}` + await resource.createVideoRoomResource(roomName) + + await createCFClient(page) + + // Dial an address and join a video room + const ms = await page.evaluate( + async ({ address }) => { + // @ts-expect-error + const client: SignalWireContract = window._client + + const call = await client.dial({ + to: address, + rootElement: document.getElementById('rootElement'), + }) + // @ts-expect-error + window._roomObj = call + + const t0 = performance.now() + await call.start() + + return performance.now() - t0 + }, + { + address: `/public/${roomName}?channel=video`, + } + ) + + console.log('::group::CallFabric perf') + console.log(`[PERF] call.start(ms)= ${ms.toFixed(0)}`) + console.log(`::notice title=Call setup latency::${Math.round(ms)} ms`) + console.log('::endgroup::') + + expect(ms).toBeLessThan(MAX_CALL_SETUP_TIME_MS) + }) + + test('should join an audio-only room within 5 seconds', async ({ + createCustomPage, + resource, + }) => { + const page = await createCustomPage({ name: '[page]' }) + await page.goto(SERVER_URL) + + const roomName = `e2e_${uuid()}` + await resource.createVideoRoomResource(roomName) + + await createCFClient(page) + + // Dial an address and join a video room + const ms = await page.evaluate( + async ({ address }) => { + // @ts-expect-error + const client: SignalWireContract = window._client + + const call = await client.dial({ + to: address, + }) + // @ts-expect-error + window._roomObj = call + + const t0 = performance.now() + await call.start() + + return performance.now() - t0 + }, + { + address: `/public/${roomName}?channel=audio`, + } + ) + + console.log('::group::CallFabric perf') + console.log(`[PERF] call.start(ms)= ${ms.toFixed(0)}`) + console.log(`::notice title=Call setup latency::${Math.round(ms)} ms`) + console.log('::endgroup::') + + expect(ms).toBeLessThan(MAX_CALL_SETUP_TIME_MS) + }) +}) diff --git a/internal/e2e-js/tests/roomSessionStartTime.spec.ts b/internal/e2e-js/tests/roomSessionStartTime.spec.ts new file mode 100644 index 000000000..f6a4ef27c --- /dev/null +++ b/internal/e2e-js/tests/roomSessionStartTime.spec.ts @@ -0,0 +1,50 @@ +import type { Video } from '@signalwire/js' +import { test, expect } from '../fixtures' +import { SERVER_URL, createTestRoomSession, randomizeRoomName } from '../utils' + +const MAX_CALL_SETUP_TIME_MS = 5000 + +test.describe('RoomSession Start Time', () => { + test('should join a room room within 5 seconds', async ({ + createCustomPage, + }) => { + const page = await createCustomPage({ name: 'raise-lower' }) + await page.goto(SERVER_URL) + + const roomName = randomizeRoomName('raise-lower-e2e') + const memberSettings = { + vrt: { + room_name: roomName, + user_name: 'e2e_participant_meta', + auto_create_room: true, + permissions: ['room.prioritize_handraise'], + }, + initialEvents: ['room.updated'], + } + + await createTestRoomSession(page, memberSettings) + + // --------------- Joining the room --------------- + const ms = await page.evaluate(() => { + return new Promise(async (resolve, reject) => { + // @ts-expect-error + const roomObj: Video.RoomSession = window._roomObj + + roomObj.once('room.joined', () => { + console.log('Room joined!') + resolve(performance.now() - t0) + }) + + const t0 = performance.now() + await roomObj.join().catch(reject) + }) + }) + + console.log('::group::CallFabric perf') + console.log(`[PERF] call.start(ms)= ${ms.toFixed(0)}`) + console.log(`::notice title=Call setup latency::${Math.round(ms)} ms`) + console.log('::endgroup::') + + expect(ms).toBeLessThan(MAX_CALL_SETUP_TIME_MS) + }) +}) diff --git a/internal/e2e-js/utils.ts b/internal/e2e-js/utils.ts index 9f4e4129c..79f54599b 100644 --- a/internal/e2e-js/utils.ts +++ b/internal/e2e-js/utils.ts @@ -1,3 +1,12 @@ +import express, { Express, Request, Response } from 'express' +import { spawn, ChildProcessWithoutNullStreams } from 'child_process' +import path from 'path' +import { EventEmitter } from 'events' +import { v4 as uuid } from 'uuid' +import { clearInterval } from 'timers' +import { Server } from 'http' +import { createServer } from 'vite' +import { Page } from '@playwright/test' import type { DialParams, FabricRoomSession, @@ -7,16 +16,8 @@ import type { Video, } from '@signalwire/js' import type { MediaEventNames } from '@signalwire/webrtc' -import { createServer } from 'vite' -import path from 'path' import { expect } from './fixtures' -import { Page } from '@playwright/test' -import { v4 as uuid } from 'uuid' -import { clearInterval } from 'timers' -import express, { Express, Request, Response } from 'express' -import { Server } from 'http' -import { spawn, ChildProcessWithoutNullStreams } from 'child_process' -import { EventEmitter } from 'events' + declare global { interface Window { _SWJS: { From 3ed10c59a5bf0170ebcd9e3ef33a6cbe1433388d Mon Sep 17 00:00:00 2001 From: Ammar Ansari Date: Tue, 12 Aug 2025 16:35:40 +0200 Subject: [PATCH 2/3] make time 4 seconds --- .../tests/callfabric/callStartTime.spec.ts | 26 ++++++++++++++----- .../e2e-js/tests/roomSessionStartTime.spec.ts | 4 +-- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/internal/e2e-js/tests/callfabric/callStartTime.spec.ts b/internal/e2e-js/tests/callfabric/callStartTime.spec.ts index bdfa435ca..b696b88dd 100644 --- a/internal/e2e-js/tests/callfabric/callStartTime.spec.ts +++ b/internal/e2e-js/tests/callfabric/callStartTime.spec.ts @@ -2,11 +2,12 @@ import { uuid } from '@signalwire/core' import { test, expect } from '../../fixtures' import { SERVER_URL, createCFClient } from '../../utils' import { SignalWireContract } from '@signalwire/js' +import { appendFileSync } from 'fs' -const MAX_CALL_SETUP_TIME_MS = 5000 +const MAX_CALL_SETUP_TIME_MS = 4000 test.describe('CallFabric Start Time', () => { - test('should join a video room within 5 seconds', async ({ + test('should join a video room within 4 seconds', async ({ createCustomPage, resource, }) => { @@ -41,15 +42,26 @@ test.describe('CallFabric Start Time', () => { } ) - console.log('::group::CallFabric perf') - console.log(`[PERF] call.start(ms)= ${ms.toFixed(0)}`) - console.log(`::notice title=Call setup latency::${Math.round(ms)} ms`) - console.log('::endgroup::') + if (ms < MAX_CALL_SETUP_TIME_MS) { + console.log(`\x1b[1;32m✅ call.start(): ${ms.toFixed(0)} ms\x1b[0m`) + } else { + console.log(`\x1b[1;31m❌ call.start(): ${ms.toFixed(0)} ms\x1b[0m`) + } + + if (process.env.GITHUB_STEP_SUMMARY) { + const summary = [ + '### CallFabric Performance', + '', + `- Room: \`${roomName}\``, + `- \`call.start()\`: **${ms.toFixed(0)} ms**`, + ].join('\n') + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `\n${summary}\n`) + } expect(ms).toBeLessThan(MAX_CALL_SETUP_TIME_MS) }) - test('should join an audio-only room within 5 seconds', async ({ + test('should join an audio-only room within 4 seconds', async ({ createCustomPage, resource, }) => { diff --git a/internal/e2e-js/tests/roomSessionStartTime.spec.ts b/internal/e2e-js/tests/roomSessionStartTime.spec.ts index f6a4ef27c..5b58700c5 100644 --- a/internal/e2e-js/tests/roomSessionStartTime.spec.ts +++ b/internal/e2e-js/tests/roomSessionStartTime.spec.ts @@ -2,10 +2,10 @@ import type { Video } from '@signalwire/js' import { test, expect } from '../fixtures' import { SERVER_URL, createTestRoomSession, randomizeRoomName } from '../utils' -const MAX_CALL_SETUP_TIME_MS = 5000 +const MAX_CALL_SETUP_TIME_MS = 4000 test.describe('RoomSession Start Time', () => { - test('should join a room room within 5 seconds', async ({ + test('should join a room room within 4 seconds', async ({ createCustomPage, }) => { const page = await createCustomPage({ name: 'raise-lower' }) From 70d3604d87d88c9f7456e9ff82daae90f411eef9 Mon Sep 17 00:00:00 2001 From: Ammar Ansari Date: Tue, 12 Aug 2025 16:51:26 +0200 Subject: [PATCH 3/3] make time 5 seconds --- .../tests/callfabric/callStartTime.spec.ts | 36 +++++++------------ .../e2e-js/tests/roomSessionStartTime.spec.ts | 17 +++++---- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/internal/e2e-js/tests/callfabric/callStartTime.spec.ts b/internal/e2e-js/tests/callfabric/callStartTime.spec.ts index b696b88dd..7bf9faea2 100644 --- a/internal/e2e-js/tests/callfabric/callStartTime.spec.ts +++ b/internal/e2e-js/tests/callfabric/callStartTime.spec.ts @@ -2,12 +2,19 @@ import { uuid } from '@signalwire/core' import { test, expect } from '../../fixtures' import { SERVER_URL, createCFClient } from '../../utils' import { SignalWireContract } from '@signalwire/js' -import { appendFileSync } from 'fs' -const MAX_CALL_SETUP_TIME_MS = 4000 +export const MAX_CALL_SETUP_TIME_MS = 5000 + +export const logCallStartTime = (ms: number) => { + if (ms < MAX_CALL_SETUP_TIME_MS) { + console.log(`\x1b[1;32m✅ call.start(): ${ms.toFixed(0)} ms\x1b[0m`) + } else { + console.log(`\x1b[1;31m❌ call.start(): ${ms.toFixed(0)} ms\x1b[0m`) + } +} test.describe('CallFabric Start Time', () => { - test('should join a video room within 4 seconds', async ({ + test('should join a video room within 5 seconds', async ({ createCustomPage, resource, }) => { @@ -42,26 +49,12 @@ test.describe('CallFabric Start Time', () => { } ) - if (ms < MAX_CALL_SETUP_TIME_MS) { - console.log(`\x1b[1;32m✅ call.start(): ${ms.toFixed(0)} ms\x1b[0m`) - } else { - console.log(`\x1b[1;31m❌ call.start(): ${ms.toFixed(0)} ms\x1b[0m`) - } - - if (process.env.GITHUB_STEP_SUMMARY) { - const summary = [ - '### CallFabric Performance', - '', - `- Room: \`${roomName}\``, - `- \`call.start()\`: **${ms.toFixed(0)} ms**`, - ].join('\n') - appendFileSync(process.env.GITHUB_STEP_SUMMARY, `\n${summary}\n`) - } + logCallStartTime(ms) expect(ms).toBeLessThan(MAX_CALL_SETUP_TIME_MS) }) - test('should join an audio-only room within 4 seconds', async ({ + test('should join an audio-only room within 5 seconds', async ({ createCustomPage, resource, }) => { @@ -95,10 +88,7 @@ test.describe('CallFabric Start Time', () => { } ) - console.log('::group::CallFabric perf') - console.log(`[PERF] call.start(ms)= ${ms.toFixed(0)}`) - console.log(`::notice title=Call setup latency::${Math.round(ms)} ms`) - console.log('::endgroup::') + logCallStartTime(ms) expect(ms).toBeLessThan(MAX_CALL_SETUP_TIME_MS) }) diff --git a/internal/e2e-js/tests/roomSessionStartTime.spec.ts b/internal/e2e-js/tests/roomSessionStartTime.spec.ts index 5b58700c5..f64f32a69 100644 --- a/internal/e2e-js/tests/roomSessionStartTime.spec.ts +++ b/internal/e2e-js/tests/roomSessionStartTime.spec.ts @@ -2,10 +2,18 @@ import type { Video } from '@signalwire/js' import { test, expect } from '../fixtures' import { SERVER_URL, createTestRoomSession, randomizeRoomName } from '../utils' -const MAX_CALL_SETUP_TIME_MS = 4000 +export const MAX_CALL_SETUP_TIME_MS = 5000 + +export const logCallStartTime = (ms: number) => { + if (ms < MAX_CALL_SETUP_TIME_MS) { + console.log(`\x1b[1;32m✅ call.start(): ${ms.toFixed(0)} ms\x1b[0m`) + } else { + console.log(`\x1b[1;31m❌ call.start(): ${ms.toFixed(0)} ms\x1b[0m`) + } +} test.describe('RoomSession Start Time', () => { - test('should join a room room within 4 seconds', async ({ + test('should join a room room within 5 seconds', async ({ createCustomPage, }) => { const page = await createCustomPage({ name: 'raise-lower' }) @@ -40,10 +48,7 @@ test.describe('RoomSession Start Time', () => { }) }) - console.log('::group::CallFabric perf') - console.log(`[PERF] call.start(ms)= ${ms.toFixed(0)}`) - console.log(`::notice title=Call setup latency::${Math.round(ms)} ms`) - console.log('::endgroup::') + logCallStartTime(ms) expect(ms).toBeLessThan(MAX_CALL_SETUP_TIME_MS) })