feat: extract video metadata client-side with mediabunny

Author: jtydhr88

.github/workflows/ci-oss-assets-validation.yaml

@@ -112,7 +112,7 @@ jobs:
             --summary \
             --excludePackages '@comfyorg/comfyui-frontend;@comfyorg/design-system;@comfyorg/ingest-types;@comfyorg/registry-types;@comfyorg/shared-frontend-utils;@comfyorg/tailwind-utils;@comfyorg/comfyui-electron-types' \
             --clarificationsFile .github/license-clarifications.json \
-            --onlyAllow 'MIT;MIT*;Apache-2.0;BSD-2-Clause;BSD-3-Clause;ISC;0BSD;BlueOak-1.0.0;Python-2.0;CC0-1.0;Unlicense;(MIT OR Apache-2.0);(MIT OR GPL-3.0);(Apache-2.0 OR MIT);(MPL-2.0 OR Apache-2.0);CC-BY-4.0;CC-BY-3.0;GPL-3.0-only'; then
+            --onlyAllow 'MIT;MIT*;Apache-2.0;BSD-2-Clause;BSD-3-Clause;ISC;0BSD;BlueOak-1.0.0;Python-2.0;CC0-1.0;Unlicense;(MIT OR Apache-2.0);(MIT OR GPL-3.0);(Apache-2.0 OR MIT);(MPL-2.0 OR Apache-2.0);MPL-2.0;CC-BY-4.0;CC-BY-3.0;GPL-3.0-only'; then
             echo ''
             echo '✅ All production dependency licenses are approved!'
           else

package.json

@@ -119,6 +119,7 @@
     "jsonata": "catalog:",
     "loglevel": "^1.9.2",
     "marked": "^15.0.11",
+    "mediabunny": "catalog:",
     "minisearch": "catalog:",
     "pinia": "catalog:",
     "posthog-js": "catalog:",

pnpm-lock.yaml

@@ -102,6 +102,7 @@ catalog:
   lenis: ^1.3.21
   lint-staged: ^16.2.7
   markdown-table: ^3.0.4
+  mediabunny: ^1.53.1
   minisearch: ^7.2.0
   mixpanel-browser: ^2.71.0
   monocart-coverage-reports: ^2.12.9

pnpm-workspace.yaml

@@ -267,7 +267,7 @@ export function useVideoFilmstrip(
       width.value = metadata?.width ?? video.videoWidth
       height.value = metadata?.height ?? video.videoHeight
       fps.value = metadata?.fps ?? options.fps ?? DEFAULT_VIDEO_FPS
-      fileSize.value = metadata?.size
+      fileSize.value = metadata?.size ?? undefined
       totalFrames.value =
         metadata?.frame_count ??
         Math.max(Math.round(effectiveDuration * fps.value), 1)

src/composables/video/useVideoFilmstrip.ts

@@ -1,75 +1,237 @@
-import { describe, expect, it, vi } from 'vitest'
+import { BufferSource } from 'mediabunny'
+import { readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it, vi } from 'vitest'
 
-import { api } from '@/scripts/api'
-import { fetchVideoMetadata } from '@/utils/videoMetadataUtil'
+import {
+  extractVideoMetadata,
+  fetchVideoMetadata,
+  snapToStandardFrameRate
+} from '@/utils/videoMetadataUtil'
 
 vi.mock('@/scripts/api', () => ({
   api: {
-    fetchApi: vi.fn(),
     apiURL: (path: string) => `http://localhost:8188/api${path}`
   }
 }))
 
-const metadata = {
-  fps: 30,
-  duration: 4.2,
-  frame_count: 126,
-  width: 1920,
-  height: 1080,
-  size: 1024
+function bufferSource(bytes: Uint8Array) {
+  return new BufferSource(bytes)
 }
 
-function mockResponse(ok: boolean, body?: unknown) {
-  vi.mocked(api.fetchApi).mockResolvedValueOnce({
-    ok,
-    json: async () => body
-  } as Response)
+function readFixture(name: string): Uint8Array {
+  return new Uint8Array(
+    readFileSync(join(process.cwd(), 'src', 'utils', '__fixtures__', name))
+  )
 }
 
-describe('fetchVideoMetadata', () => {
-  it('queries the backend with the view resource params', async () => {
-    mockResponse(true, metadata)
+describe('extractVideoMetadata', () => {
+  it('reads dimensions, duration, fps and size from an mp4', async () => {
+    const bytes = readFixture('tiny.mp4')
+
+    const result = await extractVideoMetadata(bufferSource(bytes))
+
+    expect(result).toBeDefined()
+    expect(result?.width).toBe(64)
+    expect(result?.height).toBe(48)
+    expect(result?.duration).toBeCloseTo(12 / 8, 1)
+    expect(result?.fps).toBeCloseTo(8, 1)
+    expect(result?.size).toBe(bytes.byteLength)
+  })
+
+  it('reads a webm container', async () => {
+    const bytes = readFixture('tiny.webm')
+
+    const result = await extractVideoMetadata(bufferSource(bytes))
+
+    expect(result).toBeDefined()
+    expect(result?.width).toBe(64)
+    expect(result?.height).toBe(48)
+    expect(result?.duration).toBeCloseTo(12 / 8, 1)
+    expect(result?.fps).toBeCloseTo(8, 1)
+  })
+
+  it('returns undefined for non-video bytes', async () => {
+    const bytes = new TextEncoder().encode('not actually a video file')
+
+    const result = await extractVideoMetadata(bufferSource(bytes))
+
+    expect(result).toBeUndefined()
+  })
+
+  it('returns undefined when the signal is already aborted', async () => {
+    const controller = new AbortController()
+    controller.abort()
+
+    const result = await extractVideoMetadata(
+      bufferSource(readFixture('tiny.mp4')),
+      controller.signal
+    )
+
+    expect(result).toBeUndefined()
+  })
+
+  it('reports a null size when the source size is unknown', async () => {
+    const source = bufferSource(readFixture('tiny.mp4'))
+    source.getSizeOrNull = async () => null
+
+    const result = await extractVideoMetadata(source)
+
+    expect(result).toBeDefined()
+    expect(result?.size).toBeNull()
+    expect(result?.width).toBe(64)
+  })
+})
+
+describe('snapToStandardFrameRate', () => {
+  it('snaps near-standard measurements to the exact rate', () => {
+    expect(snapToStandardFrameRate(29.972)).toBe(30_000 / 1_001)
+    expect(snapToStandardFrameRate(23.98)).toBe(24_000 / 1_001)
+    expect(snapToStandardFrameRate(30.005)).toBe(30)
+    expect(snapToStandardFrameRate(59.945)).toBe(60_000 / 1_001)
+  })
+
+  it('leaves non-standard rates unchanged', () => {
+    expect(snapToStandardFrameRate(8)).toBe(8)
+    expect(snapToStandardFrameRate(33.3)).toBe(33.3)
+  })
+})
+
+describe('fetchVideoMetadata url gating', () => {
+  afterEach(() => {
+    vi.unstubAllGlobals()
+  })
+
+  it('extracts metadata from a trusted view url', async () => {
+    const bytes = readFixture('tiny.mp4')
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(async () => new Response(new Uint8Array(bytes).buffer))
+    )
 
     const result = await fetchVideoMetadata(
-      'http://localhost:8188/api/view?filename=a.mp4&subfolder=clips&type=input&rand=0.1'
+      'http://localhost:8188/api/view?filename=tiny.mp4&type=input'
     )
 
-    expect(api.fetchApi).toHaveBeenCalledWith(
-      '/video_metadata?filename=a.mp4&subfolder=clips&type=input',
-      { signal: undefined }
+    expect(result).toBeDefined()
+    expect(result?.width).toBe(64)
+    expect(result?.height).toBe(48)
+    expect(result?.fps).toBeCloseTo(8, 1)
+  })
+
+  it('caches metadata per view resource ignoring cache-busting params', async () => {
+    const fetchMock = vi.fn(
+      async () => new Response(new Uint8Array(readFixture('tiny.mp4')).buffer)
+    )
+    vi.stubGlobal('fetch', fetchMock)
+
+    const first = await fetchVideoMetadata(
+      'http://localhost:8188/api/view?filename=cached.mp4&type=input&rand=0.1'
     )
-    expect(result).toEqual(metadata)
+    const second = await fetchVideoMetadata(
+      'http://localhost:8188/api/view?filename=cached.mp4&type=input&rand=0.2'
+    )
+
+    expect(first).toBeDefined()
+    expect(second).toEqual(first)
+    expect(fetchMock).toHaveBeenCalledTimes(1)
   })
 
-  it('returns undefined for non-view urls without fetching', async () => {
-    const result = await fetchVideoMetadata('blob:abc')
+  it('does not share cache across different origins or paths', async () => {
+    const fetchMock = vi.fn(
+      async () => new Response(new Uint8Array(readFixture('tiny.mp4')).buffer)
+    )
+    vi.stubGlobal('fetch', fetchMock)
+
+    const fromApiBase = await fetchVideoMetadata(
+      'http://localhost:8188/api/view?filename=origins.mp4&type=input'
+    )
+    const fromWindowOrigin = await fetchVideoMetadata(
+      '/api/view?filename=origins.mp4&type=input'
+    )
 
-    expect(api.fetchApi).not.toHaveBeenCalled()
-    expect(result).toBeUndefined()
+    expect(fromApiBase).toBeDefined()
+    expect(fromWindowOrigin).toBeDefined()
+    expect(fetchMock).toHaveBeenCalledTimes(2)
   })
 
-  it('rejects view urls from untrusted origins without fetching', async () => {
-    const result = await fetchVideoMetadata(
-      'https://attacker.invalid/view?filename=a.mp4'
+  it('deduplicates concurrent probes for the same resource', async () => {
+    const fetchMock = vi.fn(
+      async () => new Response(new Uint8Array(readFixture('tiny.mp4')).buffer)
     )
+    vi.stubGlobal('fetch', fetchMock)
+
+    const url =
+      'http://localhost:8188/api/view?filename=concurrent.mp4&type=input'
+    const [first, second] = await Promise.all([
+      fetchVideoMetadata(url),
+      fetchVideoMetadata(url)
+    ])
+
+    expect(first).toBeDefined()
+    expect(second).toEqual(first)
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+  })
 
-    expect(api.fetchApi).not.toHaveBeenCalled()
-    expect(result).toBeUndefined()
+  it('unblocks an aborted caller while the shared probe continues', async () => {
+    let releaseFetch!: () => void
+    const gate = new Promise<void>((resolve) => {
+      releaseFetch = resolve
+    })
+    const fetchMock = vi.fn(async () => {
+      await gate
+      return new Response(new Uint8Array(readFixture('tiny.mp4')).buffer)
+    })
+    vi.stubGlobal('fetch', fetchMock)
+
+    const url = 'http://localhost:8188/api/view?filename=aborted.mp4&type=input'
+    const controller = new AbortController()
+    const pending = fetchVideoMetadata(url, controller.signal)
+    controller.abort()
+
+    expect(await pending).toBeUndefined()
+
+    releaseFetch()
+    const result = await fetchVideoMetadata(url)
+    expect(result).toBeDefined()
+    expect(fetchMock).toHaveBeenCalledTimes(1)
   })
 
-  it('returns undefined when the backend lacks the endpoint', async () => {
-    mockResponse(false)
+  it('does not cache failed probes', async () => {
+    const failing = vi.fn(async () => {
+      throw new Error('network down')
+    })
+    vi.stubGlobal('fetch', failing)
 
-    const result = await fetchVideoMetadata('/api/view?filename=a.mp4')
+    const url = 'http://localhost:8188/api/view?filename=flaky.mp4&type=input'
+    expect(await fetchVideoMetadata(url)).toBeUndefined()
 
-    expect(result).toBeUndefined()
+    const working = vi.fn(
+      async () => new Response(new Uint8Array(readFixture('tiny.mp4')).buffer)
+    )
+    vi.stubGlobal('fetch', working)
+
+    const result = await fetchVideoMetadata(url)
+    expect(result).toBeDefined()
+    expect(working).toHaveBeenCalled()
   })
 
-  it('returns undefined for malformed responses', async () => {
-    mockResponse(true, { unexpected: true })
+  it('returns undefined for non-view urls', async () => {
+    expect(await fetchVideoMetadata('blob:abc')).toBeUndefined()
+    expect(
+      await fetchVideoMetadata('http://localhost:8188/api/other?filename=a.mp4')
+    ).toBeUndefined()
+  })
 
-    const result = await fetchVideoMetadata('/api/view?filename=a.mp4')
+  it('returns undefined for view urls without a filename', async () => {
+    expect(
+      await fetchVideoMetadata('http://localhost:8188/api/view?type=input')
+    ).toBeUndefined()
+  })
 
-    expect(result).toBeUndefined()
+  it('rejects view urls from untrusted origins', async () => {
+    expect(
+      await fetchVideoMetadata('https://attacker.invalid/view?filename=a.mp4')
+    ).toBeUndefined()
   })
 })